Merge pull request #2599 from fuzeman/tv_searcher

[TV] Moved matcher to core/media, updated NZBIndex provider

couchpotato/core/media/_base/matcher/base.py

@@ -0,0 +1,84 @@
+from couchpotato.core.event import addEvent
+from couchpotato.core.helpers.encoding import simplifyString
+from couchpotato.core.logger import CPLog
+from couchpotato.core.plugins.base import Plugin
+
+log = CPLog(__name__)
+
+
+class MatcherBase(Plugin):
+    type = None
+
+    def __init__(self):
+        if self.type:
+            addEvent('%s.matcher.correct' % self.type, self.correct)
+
+    def correct(self, chain, release, media, quality):
+        raise NotImplementedError()
+
+    def flattenInfo(self, info):
+        # Flatten dictionary of matches (chain info)
+        if isinstance(info, dict):
+            return dict([(key, self.flattenInfo(value)) for key, value in info.items()])
+
+        # Flatten matches
+        result = None
+
+        for match in info:
+            if isinstance(match, dict):
+                if result is None:
+                    result = {}
+
+                for key, value in match.items():
+                    if key not in result:
+                        result[key] = []
+
+                    result[key].append(value)
+            else:
+                if result is None:
+                    result = []
+
+                result.append(match)
+
+        return result
+
+    def constructFromRaw(self, match):
+        if not match:
+            return None
+
+        parts = [
+            ''.join([
+                y for y in x[1:] if y
+            ]) for x in match
+        ]
+
+        return ''.join(parts)[:-1].strip()
+
+    def simplifyValue(self, value):
+        if not value:
+            return value
+
+        if isinstance(value, basestring):
+            return simplifyString(value)
+
+        if isinstance(value, list):
+            return [self.simplifyValue(x) for x in value]
+
+        raise ValueError("Unsupported value type")
+
+    def chainMatch(self, chain, group, tags):
+        info = self.flattenInfo(chain.info[group])
+
+        found_tags = []
+        for tag, accepted in tags.items():
+            values = [self.simplifyValue(x) for x in info.get(tag, [None])]
+
+            if any([val in accepted for val in values]):
+                found_tags.append(tag)
+
+        log.debug('tags found: %s, required: %s' % (found_tags, tags.keys()))
+
+        if set(tags.keys()) == set(found_tags):
+            return True
+
+        return all([key in found_tags for key, value in tags.items()])

couchpotato/core/media/_base/matcher/main.py

@@ -0,0 +1,88 @@
+from couchpotato.core.event import addEvent, fireEvent
+from couchpotato.core.helpers.variable import possibleTitles
+from couchpotato.core.logger import CPLog
+from couchpotato.core.media._base.matcher.base import MatcherBase
+from caper import Caper
+
+log = CPLog(__name__)
+
+
+class Matcher(MatcherBase):
+    def __init__(self):
+        super(Matcher, self).__init__()
+
+        self.caper = Caper()
+
+        addEvent('matcher.parse', self.parse)
+        addEvent('matcher.match', self.match)
+
+        addEvent('matcher.flatten_info', self.flattenInfo)
+        addEvent('matcher.construct_from_raw', self.constructFromRaw)
+
+        addEvent('matcher.correct_title', self.correctTitle)
+        addEvent('matcher.correct_quality', self.correctQuality)
+
+    def parse(self, name, parser='scene'):
+        return self.caper.parse(name, parser)
+
+    def match(self, release, media, quality):
+        match = fireEvent('matcher.parse', release['name'], single = True)
+
+        if len(match.chains) < 1:
+            log.info2('Wrong: %s, unable to parse release name (no chains)', release['name'])
+            return False
+
+        for chain in match.chains:
+            if fireEvent('%s.matcher.correct' % media['type'], chain, release, media, quality, single = True):
+                return chain
+
+        return False
+
+    def correctTitle(self, chain, media):
+        root_library = media['library']['root_library']
+
+        if 'show_name' not in chain.info or not len(chain.info['show_name']):
+            log.info('Wrong: missing show name in parsed result')
+            return False
+
+        # Get the lower-case parsed show name from the chain
+        chain_words = [x.lower() for x in chain.info['show_name']]
+
+        # Build a list of possible titles of the media we are searching for
+        titles = root_library['info']['titles']
+
+        # Add year suffix titles (will result in ['<name_one>', '<name_one> <suffix_one>', '<name_two>', ...])
+        suffixes = [None, root_library['info']['year']]
+
+        titles = [
+            title + ((' %s' % suffix) if suffix else '')
+            for title in titles
+            for suffix in suffixes
+        ]
+
+        # Check show titles match
+        # TODO check xem names
+        for title in titles:
+            for valid_words in [x.split(' ') for x in possibleTitles(title)]:
+
+                if valid_words == chain_words:
+                    return True
+
+        return False
+
+    def correctQuality(self, chain, quality, quality_map):
+        if quality['identifier'] not in quality_map:
+            log.info2('Wrong: unknown preferred quality %s', quality['identifier'])
+            return False
+
+        if 'video' not in chain.info:
+            log.info2('Wrong: no video tags found')
+            return False
+
+        video_tags = quality_map[quality['identifier']]
+
+        if not self.chainMatch(chain, 'video', video_tags):
+            log.info2('Wrong: %s tags not in chain', video_tags)
+            return False
+
+        return True

couchpotato/core/media/show/library/season/main.py

@@ -26,11 +26,8 @@ class SeasonLibraryPlugin(LibraryBase):
         if library.get('type') != 'season':
             return
 
-        season_num = tryInt(library['season_number'], None)
-
         return {
-            'season': season_num,
+            'season': tryInt(library['season_number'], None)
-            'episode': None
         }
 
     def add(self, attrs = {}, update_after = True):

couchpotato/core/media/show/matcher/__init__.py

@@ -0,0 +1,6 @@
+from .main import ShowMatcher
+
+def start():
+    return ShowMatcher()
+
+config = []

couchpotato/core/media/show/matcher/main.py

@@ -0,0 +1,127 @@
+from couchpotato import CPLog
+from couchpotato.core.event import addEvent, fireEvent
+from couchpotato.core.helpers.variable import dictIsSubset, tryInt, toIterable
+from couchpotato.core.media._base.matcher.base import MatcherBase
+from couchpotato.core.providers.base import MultiProvider
+
+log = CPLog(__name__)
+
+
+class ShowMatcher(MultiProvider):
+
+    def getTypes(self):
+        return [Season, Episode]
+
+
+class Base(MatcherBase):
+    # TODO come back to this later, think this could be handled better, this is starting to get out of hand....
+    quality_map = {
+        'bluray_1080p': {'resolution': ['1080p'], 'source': ['bluray']},
+        'bluray_720p': {'resolution': ['720p'], 'source': ['bluray']},
+
+        'bdrip_1080p': {'resolution': ['1080p'], 'source': ['BDRip']},
+        'bdrip_720p': {'resolution': ['720p'], 'source': ['BDRip']},
+
+        'brrip_1080p': {'resolution': ['1080p'], 'source': ['BRRip']},
+        'brrip_720p': {'resolution': ['720p'], 'source': ['BRRip']},
+
+        'webdl_1080p': {'resolution': ['1080p'], 'source': ['webdl', ['web', 'dl']]},
+        'webdl_720p': {'resolution': ['720p'], 'source': ['webdl', ['web', 'dl']]},
+        'webdl_480p': {'resolution': ['480p'], 'source': ['webdl', ['web', 'dl']]},
+
+        'hdtv_720p': {'resolution': ['720p'], 'source': ['hdtv']},
+        'hdtv_sd': {'resolution': ['480p', None], 'source': ['hdtv']},
+    }
+
+    def __init__(self):
+        super(Base, self).__init__()
+
+        addEvent('%s.matcher.correct_identifier' % self.type, self.correctIdentifier)
+
+    def correct(self, chain, release, media, quality):
+        log.info("Checking if '%s' is valid", release['name'])
+        log.info2('Release parsed as: %s', chain.info)
+
+        if not fireEvent('matcher.correct_quality', chain, quality, self.quality_map, single = True):
+            log.info('Wrong: %s, quality does not match', release['name'])
+            return False
+
+        if not fireEvent('%s.matcher.correct_identifier' % self.type, chain, media):
+            log.info('Wrong: %s, identifier does not match', release['name'])
+            return False
+
+        if not fireEvent('matcher.correct_title', chain, media):
+            log.info("Wrong: '%s', undetermined naming.", (' '.join(chain.info['show_name'])))
+            return False
+
+        return True
+
+    def correctIdentifier(self, chain, media):
+        raise NotImplementedError()
+
+    def getChainIdentifier(self, chain):
+        if 'identifier' not in chain.info:
+            return None
+
+        identifier = self.flattenInfo(chain.info['identifier'])
+
+        # Try cast values to integers
+        for key, value in identifier.items():
+            if isinstance(value, list):
+                if len(value) <= 1:
+                    value = value[0]
+                else:
+                    log.warning('Wrong: identifier contains multiple season or episode values, unsupported')
+                    return None
+
+            identifier[key] = tryInt(value, value)
+
+        return identifier
+
+
+class Episode(Base):
+    type = 'episode'
+
+    def correctIdentifier(self, chain, media):
+        identifier = self.getChainIdentifier(chain)
+        if not identifier:
+            log.info2('Wrong: release identifier is not valid (unsupported or missing identifier)')
+            return False
+
+        # TODO - Parse episode ranges from identifier to determine if they are multi-part episodes
+        if any([x in identifier for x in ['episode_from', 'episode_to']]):
+            log.info2('Wrong: releases with identifier ranges are not supported yet')
+            return False
+
+        required = fireEvent('library.identifier', media['library'], single = True)
+
+        # TODO - Support air by date episodes
+        # TODO - Support episode parts
+
+        if identifier != required:
+            log.info2('Wrong: required identifier (%s) does not match release identifier (%s)', (required, identifier))
+            return False
+
+        return True
+
+class Season(Base):
+    type = 'season'
+
+    def correctIdentifier(self, chain, media):
+        identifier = self.getChainIdentifier(chain)
+        if not identifier:
+            log.info2('Wrong: release identifier is not valid (unsupported or missing identifier)')
+            return False
+
+        # TODO - Parse episode ranges from identifier to determine if they are season packs
+        if any([x in identifier for x in ['episode_from', 'episode_to']]):
+            log.info2('Wrong: releases with identifier ranges are not supported yet')
+            return False
+
+        required = fireEvent('library.identifier', media['library'], single = True)
+
+        if identifier != required:
+            log.info2('Wrong: required identifier (%s) does not match release identifier (%s)', (required, identifier))
+            return False
+
+        return True

couchpotato/core/media/show/searcher/main.py

@@ -18,25 +18,6 @@ class ShowSearcher(Plugin):
 
     in_progress = False
 
-    # TODO come back to this later, think this could be handled better, this is starting to get out of hand....
-    quality_map = {
-        'bluray_1080p': {'resolution': ['1080p'], 'source': ['bluray']},
-        'bluray_720p': {'resolution': ['720p'], 'source': ['bluray']},
-
-        'bdrip_1080p': {'resolution': ['1080p'], 'source': ['BDRip']},
-        'bdrip_720p': {'resolution': ['720p'], 'source': ['BDRip']},
-
-        'brrip_1080p': {'resolution': ['1080p'], 'source': ['BRRip']},
-        'brrip_720p': {'resolution': ['720p'], 'source': ['BRRip']},
-
-        'webdl_1080p': {'resolution': ['1080p'], 'source': ['webdl', ['web', 'dl']]},
-        'webdl_720p': {'resolution': ['720p'], 'source': ['webdl', ['web', 'dl']]},
-        'webdl_480p': {'resolution': ['480p'], 'source': ['webdl', ['web', 'dl']]},
-
-        'hdtv_720p': {'resolution': ['720p'], 'source': ['hdtv']},
-        'hdtv_sd': {'resolution': ['480p', None], 'source': ['hdtv']},
-    }
-
     def __init__(self):
         super(ShowSearcher, self).__init__()
 
@@ -46,8 +27,6 @@ class ShowSearcher(Plugin):
             addEvent('%s.searcher.single' % type, self.single)
 
         addEvent('searcher.get_search_title', self.getSearchTitle)
-
-        addEvent('searcher.correct_match', self.correctMatch)
         addEvent('searcher.correct_release', self.correctRelease)
 
     def single(self, media, search_protocols = None, manual = False):
@@ -95,6 +74,8 @@ class ShowSearcher(Plugin):
         fireEvent('notify.frontend', type = 'show.searcher.started.%s' % media['id'], data = True, message = 'Searching for "%s"' % default_title)
 
         ret = False
+        has_better_quality = None
+
         for quality_type in media['profile']['types']:
             # TODO check air date?
             #if not self.conf('always_search') and not self.couldBeReleased(quality_type['quality']['identifier'] in pre_releases, release_dates, movie['library']['year']):
@@ -149,7 +130,7 @@ class ShowSearcher(Plugin):
 
         if len(too_early_to_search) > 0:
             log.info2('Too early to search for %s, %s', (too_early_to_search, default_title))
-        elif media['type'] == 'season' and not ret:
+        elif media['type'] == 'season' and not ret and has_better_quality is 0:
             # If nothing was found, start searching for episodes individually
             log.info('No season pack found, starting individual episode search')
 
@@ -214,7 +195,7 @@ class ShowSearcher(Plugin):
         if identifier['season']:
             title += ' S%02d' % identifier['season']
 
-            if identifier['episode']:
+            if identifier.get('episode'):
                 title += 'E%02d' % identifier['episode']
 
         return title
@@ -234,30 +215,12 @@ class ShowSearcher(Plugin):
             return False
 
         # TODO Matching is quite costly, maybe we should be caching release matches somehow? (also look at caper optimizations)
-        match = fireEvent('matcher.best', release, media, quality, single = True)
+        match = fireEvent('matcher.match', release, media, quality, single = True)
         if match:
             return match.weight
 
         return False
 
-    def correctMatch(self, chain, release, media, quality):
-        log.info("Checking if '%s' is valid", release['name'])
-        log.info2('Release parsed as: %s', chain.info)
-
-        if not fireEvent('matcher.correct_quality', chain, quality, self.quality_map, single = True):
-            log.info('Wrong: %s, quality does not match', release['name'])
-            return False
-
-        if not fireEvent('matcher.correct_identifier', chain, media):
-            log.info('Wrong: %s, identifier does not match', release['name'])
-            return False
-
-        if not fireEvent('matcher.correct_title', chain, media):
-            log.info("Wrong: '%s', undetermined naming.", (' '.join(chain.info['show_name'])))
-            return False
-
-        return True
-
     def getLibraries(self, library):
         if 'related_libraries' not in library:
             log.warning("'related_libraries' missing from media library, unable to continue searching")

couchpotato/core/plugins/matcher/main.py

@@ -1,157 +0,0 @@
-from caper import Caper
-from couchpotato import CPLog, tryInt
-from couchpotato.core.event import addEvent, fireEvent
-from couchpotato.core.helpers.encoding import simplifyString
-from couchpotato.core.helpers.variable import possibleTitles, dictIsSubset
-from couchpotato.core.plugins.base import Plugin
-
-log = CPLog(__name__)
-
-
-class Matcher(Plugin):
-    def __init__(self):
-        self.caper = Caper()
-
-        addEvent('matcher.parse', self.parse)
-        addEvent('matcher.best', self.best)
-
-        addEvent('matcher.correct_title', self.correctTitle)
-        addEvent('matcher.correct_identifier', self.correctIdentifier)
-        addEvent('matcher.correct_quality', self.correctQuality)
-
-    def parse(self, release):
-        return self.caper.parse(release['name'])
-
-    def best(self, release, media, quality):
-        match = fireEvent('matcher.parse', release, single = True)
-
-        if len(match.chains) < 1:
-            log.info2('Wrong: %s, unable to parse release name (no chains)', release['name'])
-            return False
-
-        for chain in match.chains:
-            if fireEvent('searcher.correct_match', chain, release, media, quality, single = True):
-                return chain
-
-        return False
-
-    def flattenInfo(self, info):
-        flat_info = {}
-
-        for match in info:
-            for key, value in match.items():
-                if key not in flat_info:
-                    flat_info[key] = []
-
-                flat_info[key].append(value)
-
-        return flat_info
-
-    def simplifyValue(self, value):
-        if not value:
-            return value
-
-        if isinstance(value, basestring):
-            return simplifyString(value)
-
-        if isinstance(value, list):
-            return [self.simplifyValue(x) for x in value]
-
-        raise ValueError("Unsupported value type")
-
-    def chainMatch(self, chain, group, tags):
-        info = self.flattenInfo(chain.info[group])
-
-        found_tags = []
-        for tag, accepted in tags.items():
-            values = [self.simplifyValue(x) for x in info.get(tag, [None])]
-
-            if any([val in accepted for val in values]):
-                found_tags.append(tag)
-
-        log.debug('tags found: %s, required: %s' % (found_tags, tags.keys()))
-
-        if set(tags.keys()) == set(found_tags):
-            return True
-
-        return all([key in found_tags for key, value in tags.items()])
-
-    def correctIdentifier(self, chain, media):
-        required_id = fireEvent('library.identifier', media['library'], single = True)
-
-        if 'identifier' not in chain.info:
-            return False
-
-        # TODO could be handled better?
-        if len(chain.info['identifier']) != 1:
-            return False
-        identifier = chain.info['identifier'][0]
-
-        # TODO air by date episodes
-
-        # TODO this should support identifiers with characters 'a', 'b', etc..
-        for k, v in identifier.items():
-            identifier[k] = tryInt(v, None)
-
-        if any([x in identifier for x in ['episode_from', 'episode_to']]):
-            log.info2('Wrong: releases with identifier ranges are not supported yet')
-            return False
-
-        # 'episode' is required in identifier for subset matching
-        if 'episode' not in identifier:
-            identifier['episode'] = None
-
-        if not dictIsSubset(required_id, identifier):
-            log.info2('Wrong: required identifier %s does not match release identifier %s', (str(required_id), str(identifier)))
-            return False
-
-        return True
-
-    def correctTitle(self, chain, media):
-        root_library = media['library']['root_library']
-
-        if 'show_name' not in chain.info or not len(chain.info['show_name']):
-            log.info('Wrong: missing show name in parsed result')
-            return False
-
-        # Get the lower-case parsed show name from the chain
-        chain_words = [x.lower() for x in chain.info['show_name']]
-
-        # Build a list of possible titles of the media we are searching for
-        titles = root_library['info']['titles']
-
-        # Add year suffix titles (will result in ['<name_one>', '<name_one> <suffix_one>', '<name_two>', ...])
-        suffixes = [None, root_library['info']['year']]
-
-        titles = [
-            title + ((' %s' % suffix) if suffix else '')
-            for title in titles
-            for suffix in suffixes
-        ]
-
-        # Check show titles match
-        # TODO check xem names
-        for title in titles:
-            for valid_words in [x.split(' ') for x in possibleTitles(title)]:
-
-                if valid_words == chain_words:
-                    return True
-
-        return False
-
-    def correctQuality(self, chain, quality, quality_map):
-        if quality['identifier'] not in quality_map:
-            log.info2('Wrong: unknown preferred quality %s', quality['identifier'])
-            return False
-
-        if 'video' not in chain.info:
-            log.info2('Wrong: no video tags found')
-            return False
-
-        video_tags = quality_map[quality['identifier']]
-
-        if not self.chainMatch(chain, 'video', video_tags):
-            log.info2('Wrong: %s tags not in chain', video_tags)
-            return False
-
-        return True

couchpotato/core/providers/nzb/nzbindex/main.py

@@ -38,6 +38,35 @@ class Base(NZBProvider, RSS):
             enclosure = self.getElement(nzb, 'enclosure').attrib
             nzbindex_id = int(self.getTextElement(nzb, "link").split('/')[4])
 
+            title = self.getTextElement(nzb, "title")
+
+            match = fireEvent('matcher.parse', title, parser='usenet', single = True)
+            if not match.chains:
+                log.info('Unable to parse release with title "%s"', title)
+                continue
+
+            # TODO should we consider other lower-weight chains here?
+            info = fireEvent('matcher.flatten_info', match.chains[0].info, single = True)
+
+            release_name = fireEvent('matcher.construct_from_raw', info.get('release_name'), single = True)
+
+            file_name = info.get('detail', {}).get('file_name')
+            file_name = file_name[0] if file_name else None
+
+            title = release_name or file_name
+
+            # Strip extension from parsed title (if one exists)
+            ext_pos = title.rfind('.')
+
+            # Assume extension if smaller than 4 characters
+            # TODO this should probably be done a better way
+            if len(title[ext_pos + 1:]) <= 4:
+                title = title[:ext_pos]
+
+            if not title:
+                log.info('Unable to find release name from match')
+                continue
+
             try:
                 description = self.getTextElement(nzb, "description")
             except:
@@ -52,7 +81,7 @@ class Base(NZBProvider, RSS):
 
             results.append({
                 'id': nzbindex_id,
-                'name': self.getTextElement(nzb, "title"),
+                'name': title,
                 'age': self.calculateAge(int(time.mktime(parse(self.getTextElement(nzb, "pubDate")).timetuple()))),
                 'size': tryInt(enclosure['length']) / 1024 / 1024,
                 'url': enclosure['url'],
@@ -74,7 +103,7 @@ class Base(NZBProvider, RSS):
 
 class Movie(MovieProvider, Base):
 
-    def buildUrl(self, media):
+    def buildUrl(self, media, quality):
         title = fireEvent('searcher.get_search_title', media['library'], single = True)
         year =  media['library']['year']
 

libs/caper/__init__.py

@@ -17,9 +17,10 @@ from caper.matcher import FragmentMatcher
 from caper.objects import CaperFragment, CaperClosure
 from caper.parsers.anime import AnimeParser
 from caper.parsers.scene import SceneParser
+from caper.parsers.usenet import UsenetParser
 
 
-__version_info__ = ('0', '2', '6')
+__version_info__ = ('0', '3', '1')
 __version_branch__ = 'master'
 
 __version__ = "%s%s" % (
@@ -28,8 +29,9 @@ __version__ = "%s%s" % (
 )
 
 
-CL_START_CHARS = ['(', '[']
+CL_START_CHARS = ['(', '[', '<', '>']
-CL_END_CHARS = [')', ']']
+CL_END_CHARS = [')', ']', '<', '>']
+CL_END_STRINGS = [' - ']
 
 STRIP_START_CHARS = ''.join(CL_START_CHARS)
 STRIP_END_CHARS = ''.join(CL_END_CHARS)
@@ -44,9 +46,12 @@ CL_END = 1
 
 class Caper(object):
     def __init__(self, debug=False):
+        self.debug = debug
+
         self.parsers = {
-            'scene': SceneParser(debug),
+            'anime': AnimeParser,
-            'anime': AnimeParser(debug)
+            'scene': SceneParser,
+            'usenet': UsenetParser
         }
 
     def _closure_split(self, name):
@@ -60,10 +65,10 @@ class Caper(object):
 
         def end_closure(closures, buf):
             buf = buf.strip(STRIP_CHARS)
-            if len(buf) < 1:
+            if len(buf) < 2:
                 return
 
-            cur = CaperClosure(buf)
+            cur = CaperClosure(len(closures), buf)
             cur.left = closures[len(closures) - 1] if len(closures) > 0 else None
 
             if cur.left:
@@ -74,6 +79,7 @@ class Caper(object):
         state = CL_START
         buf = ""
         for x, ch in enumerate(name):
+            # Check for start characters
             if state == CL_START and ch in CL_START_CHARS:
                 end_closure(closures, buf)
 
@@ -83,8 +89,15 @@ class Caper(object):
             buf += ch
 
             if state == CL_END and ch in CL_END_CHARS:
+                # End character found, create the closure
                 end_closure(closures, buf)
 
+                state = CL_START
+                buf = ""
+            elif state == CL_START and buf[-3:] in CL_END_STRINGS:
+                # End string found, create the closure
+                end_closure(closures, buf[:-3])
+
                 state = CL_START
                 buf = ""
 
@@ -109,7 +122,7 @@ class Caper(object):
         """
 
         cur_position = 0
-        cur = CaperFragment()
+        cur = None
 
         def end_fragment(fragments, cur, cur_position):
             cur.position = cur_position
@@ -126,23 +139,41 @@ class Caper(object):
         for closure in closures:
             closure.fragments = []
 
+            separator_buffer = ""
+
             for x, ch in enumerate(self._clean_closure(closure.value)):
+                if not cur:
+                    cur = CaperFragment(closure)
+
                 if ch in FRAGMENT_SEPARATORS:
-                    end_fragment(closure.fragments, cur, cur_position)
+                    if cur.value:
+                        separator_buffer = ""
+
+                    separator_buffer += ch
+
+                    if cur.value or not closure.fragments:
+                        end_fragment(closure.fragments, cur, cur_position)
+                    elif len(separator_buffer) > 1:
+                        cur.value = separator_buffer.strip()
+
+                        if cur.value:
+                            end_fragment(closure.fragments, cur, cur_position)
+
+                        separator_buffer = ""
 
                     # Reset
-                    cur = CaperFragment()
+                    cur = None
                     cur_position += 1
                 else:
                     cur.value += ch
 
             # Finish parsing the last fragment
-            if cur.value != "":
+            if cur and cur.value:
                 end_fragment(closure.fragments, cur, cur_position)
 
                 # Reset
                 cur_position = 0
-                cur = CaperFragment()
+                cur = None
 
         return closures
 
@@ -154,8 +185,11 @@ class Caper(object):
         for closure in closures:
             Logr.debug("closure [%s]", closure.value)
 
+            for fragment in closure.fragments:
+                Logr.debug("\tfragment [%s]", fragment.value)
+
         if parser not in self.parsers:
             raise ValueError("Unknown parser")
 
         # TODO autodetect the parser type
-        return self.parsers[parser].run(closures)
+        return self.parsers[parser](self.debug).run(closures)

libs/caper/constraint.py

@@ -14,7 +14,7 @@
 
 
 class CaptureConstraint(object):
-    def __init__(self, capture_group, comparisons=None, **kwargs):
+    def __init__(self, capture_group, constraint_type, comparisons=None, target=None, **kwargs):
         """Capture constraint object
 
         :type capture_group: CaptureGroup
@@ -22,50 +22,113 @@ class CaptureConstraint(object):
 
         self.capture_group = capture_group
 
-        self.comparisons = comparisons if comparisons else []
+        self.constraint_type = constraint_type
+        self.target = target
 
-        for key, value in kwargs.items():
+        self.comparisons = comparisons if comparisons else []
-            key = key.split('__')
+        self.kwargs = {}
+
+        for orig_key, value in kwargs.items():
+            key = orig_key.split('__')
             if len(key) != 2:
+                self.kwargs[orig_key] = value
                 continue
             name, method = key
 
-            method = '_compare_' + method
+            method = 'constraint_match_' + method
             if not hasattr(self, method):
+                self.kwargs[orig_key] = value
                 continue
 
             self.comparisons.append((name, getattr(self, method), value))
 
-    def _compare_eq(self, fragment, name, expected):
+    def execute(self, parent_node, node, **kwargs):
-        if not hasattr(fragment, name):
+        func_name = 'constraint_%s' % self.constraint_type
-            return 1.0, False
 
-        return 1.0, getattr(fragment, name) == expected
+        if hasattr(self, func_name):
+            return getattr(self, func_name)(parent_node, node, **kwargs)
 
-    def _compare_re(self, fragment, name, arg):
+        raise ValueError('Unknown constraint type "%s"' % self.constraint_type)
-        if name == 'fragment':
-            group, minimum_weight = arg if type(arg) is tuple and len(arg) > 1 else (arg, 0)
 
-            weight, match, num_fragments = self.capture_group.parser.matcher.fragment_match(fragment, group)
+    #
-            return weight, weight > minimum_weight
+    # Node Matching
-        elif type(arg).__name__ == 'SRE_Pattern':
+    #
-            return 1.0, arg.match(getattr(fragment, name)) is not None
-        elif hasattr(fragment, name):
-            match = self.capture_group.parser.matcher.value_match(getattr(fragment, name), arg, single=True)
-            return 1.0, match is not None
-        else:
-            raise ValueError("Unable to find attribute with name '%s'" % name)
 
-    def execute(self, fragment):
+    def constraint_match(self, parent_node, node):
         results = []
         total_weight = 0
 
         for name, method, argument in self.comparisons:
-            weight, success = method(fragment, name, argument)
+            weight, success = method(node, name, argument)
             total_weight += weight
             results.append(success)
 
-        return total_weight / float(len(results)), all(results) if len(results) > 0 else False
+        return total_weight / (float(len(results)) or 1), all(results) if len(results) > 0 else False
+
+    def constraint_match_eq(self, node, name, expected):
+        if not hasattr(node, name):
+            return 1.0, False
+
+        return 1.0, getattr(node, name) == expected
+
+    def constraint_match_re(self, node, name, arg):
+        # Node match
+        if name == 'node':
+            group, minimum_weight = arg if type(arg) is tuple and len(arg) > 1 else (arg, 0)
+
+            weight, match, num_fragments = self.capture_group.parser.matcher.fragment_match(node, group)
+            return weight, weight > minimum_weight
+
+        # Regex match
+        if type(arg).__name__ == 'SRE_Pattern':
+            return 1.0, arg.match(getattr(node, name)) is not None
+
+        # Value match
+        if hasattr(node, name):
+            match = self.capture_group.parser.matcher.value_match(getattr(node, name), arg, single=True)
+            return 1.0, match is not None
+
+        raise ValueError("Unknown constraint match type '%s'" % name)
+
+    #
+    # Result
+    #
+
+    def constraint_result(self, parent_node, fragment):
+        ctag = self.kwargs.get('tag')
+        if not ctag:
+            return 0, False
+
+        ckey = self.kwargs.get('key')
+
+        for tag, result in parent_node.captured():
+            if tag != ctag:
+                continue
+
+            if not ckey or ckey in result.keys():
+                return 1.0, True
+
+        return 0.0, False
+
+    #
+    # Failure
+    #
+
+    def constraint_failure(self, parent_node, fragment, match):
+        if not match or not match.success:
+            return 1.0, True
+
+        return 0, False
+
+    #
+    # Success
+    #
+
+    def constraint_success(self, parent_node, fragment, match):
+        if match and match.success:
+            return 1.0, True
+
+        return 0, False
 
     def __repr__(self):
         return "CaptureConstraint(comparisons=%s)" % repr(self.comparisons)

libs/caper/group.py

@@ -14,7 +14,7 @@
 
 
 from logr import Logr
-from caper import CaperClosure
+from caper import CaperClosure, CaperFragment
 from caper.helpers import clean_dict
 from caper.result import CaperFragmentNode, CaperClosureNode
 from caper.step import CaptureStep
@@ -34,86 +34,214 @@ class CaptureGroup(object):
 
         #: @type: list of CaptureStep
         self.steps = []
-        #: @type: list of CaptureConstraint
-        self.constraints = []
 
-    def capture_fragment(self, tag, regex=None, func=None, single=True):
+        #: type: str
+        self.step_source = None
+
+        #: @type: list of CaptureConstraint
+        self.pre_constraints = []
+
+        #: :type: list of CaptureConstraint
+        self.post_constraints = []
+
+    def capture_fragment(self, tag, regex=None, func=None, single=True, **kwargs):
         Logr.debug('capture_fragment("%s", "%s", %s, %s)', tag, regex, func, single)
 
+        if self.step_source != 'fragment':
+            if self.step_source is None:
+                self.step_source = 'fragment'
+            else:
+                raise ValueError("Unable to mix fragment and closure capturing in a group")
+
         self.steps.append(CaptureStep(
             self, tag,
             'fragment',
             regex=regex,
             func=func,
-            single=single
+            single=single,
+            **kwargs
         ))
 
         return self
 
-    def capture_closure(self, tag, regex=None, func=None, single=True):
+    def capture_closure(self, tag, regex=None, func=None, single=True, **kwargs):
         Logr.debug('capture_closure("%s", "%s", %s, %s)', tag, regex, func, single)
 
+        if self.step_source != 'closure':
+            if self.step_source is None:
+                self.step_source = 'closure'
+            else:
+                raise ValueError("Unable to mix fragment and closure capturing in a group")
+
         self.steps.append(CaptureStep(
             self, tag,
             'closure',
             regex=regex,
             func=func,
-            single=single
+            single=single,
+            **kwargs
         ))
 
         return self
 
-    def until(self, **kwargs):
+    def until_closure(self, **kwargs):
-        self.constraints.append(CaptureConstraint(self, **kwargs))
+        self.pre_constraints.append(CaptureConstraint(self, 'match', target='closure', **kwargs))
+
+        return self
+
+    def until_fragment(self, **kwargs):
+        self.pre_constraints.append(CaptureConstraint(self, 'match', target='fragment', **kwargs))
+
+        return self
+
+    def until_result(self, **kwargs):
+        self.pre_constraints.append(CaptureConstraint(self, 'result', **kwargs))
+
+        return self
+
+    def until_failure(self, **kwargs):
+        self.post_constraints.append(CaptureConstraint(self, 'failure', **kwargs))
+
+        return self
+
+    def until_success(self, **kwargs):
+        self.post_constraints.append(CaptureConstraint(self, 'success', **kwargs))
 
         return self
 
     def parse_subject(self, parent_head, subject):
+        Logr.debug("parse_subject (%s) subject: %s", self.step_source, repr(subject))
+
+        if type(subject) is CaperClosure:
+            return self.parse_closure(parent_head, subject)
+
+        if type(subject) is CaperFragment:
+            return self.parse_fragment(parent_head, subject)
+
+        raise ValueError('Unknown subject (%s)', subject)
+
+    def parse_fragment(self, parent_head, subject):
         parent_node = parent_head[0] if type(parent_head) is list else parent_head
 
-        # TODO just jumping into closures for now, will be fixed later
+        nodes, match = self.match(parent_head, parent_node, subject)
-        if type(subject) is CaperClosure:
-            return [CaperClosureNode(subject, parent_head)]
 
-        nodes = []
+        # Capturing broke on constraint, return now
-
+        if not match:
-        # Check constraints
+            return nodes
-        for constraint in self.constraints:
-            weight, success = constraint.execute(subject)
-            if success:
-                Logr.debug('capturing broke on "%s" at %s', subject.value, constraint)
-                parent_node.finished_groups.append(self)
-                nodes.append(parent_head)
-
-                if weight == 1.0:
-                    return nodes
-                else:
-                    Logr.debug('Branching result')
-
-        # Try match subject against the steps available
-        tag, success, weight, match, num_fragments = (None, None, None, None, None)
-        for step in self.steps:
-            tag = step.tag
-            success, weight, match, num_fragments = step.execute(subject)
-            if success:
-                match = clean_dict(match) if type(match) is dict else match
-                Logr.debug('Found match with weight %s, match: %s, num_fragments: %s' % (weight, match, num_fragments))
-                break
 
         Logr.debug('created fragment node with subject.value: "%s"' % subject.value)
 
-        result = [CaperFragmentNode(parent_node.closure, subject.take_right(num_fragments), parent_head, tag, weight, match)]
+        result = [CaperFragmentNode(
+            parent_node.closure,
+            subject.take_right(match.num_fragments),
+            parent_head,
+            match
+        )]
 
-        if match and weight < 1.0:
+        # Branch if the match was indefinite (weight below 1.0)
-            if num_fragments == 1:
+        if match.result and match.weight < 1.0:
-                result.append(CaperFragmentNode(parent_node.closure, [subject], parent_head, None, None, None))
+            if match.num_fragments == 1:
+                result.append(CaperFragmentNode(parent_node.closure, [subject], parent_head))
             else:
-                nodes.append(CaperFragmentNode(parent_node.closure, [subject], parent_head, None, None, None))
+                nodes.append(CaperFragmentNode(parent_node.closure, [subject], parent_head))
 
         nodes.append(result[0] if len(result) == 1 else result)
 
         return nodes
 
+    def parse_closure(self, parent_head, subject):
+        parent_node = parent_head[0] if type(parent_head) is list else parent_head
+
+        nodes, match = self.match(parent_head, parent_node, subject)
+
+        # Capturing broke on constraint, return now
+        if not match:
+            return nodes
+
+        Logr.debug('created closure node with subject.value: "%s"' % subject.value)
+
+        result = [CaperClosureNode(
+            subject,
+            parent_head,
+            match
+        )]
+
+        # Branch if the match was indefinite (weight below 1.0)
+        if match.result and match.weight < 1.0:
+            if match.num_fragments == 1:
+                result.append(CaperClosureNode(subject, parent_head))
+            else:
+                nodes.append(CaperClosureNode(subject, parent_head))
+
+        nodes.append(result[0] if len(result) == 1 else result)
+
+        return nodes
+
+    def match(self, parent_head, parent_node, subject):
+        nodes = []
+
+        # Check pre constaints
+        broke, definite = self.check_constraints(self.pre_constraints, parent_head, subject)
+
+        if broke:
+            nodes.append(parent_head)
+
+            if definite:
+                return nodes, None
+
+        # Try match subject against the steps available
+        match = None
+
+        for step in self.steps:
+            if step.source == 'closure' and type(subject) is not CaperClosure:
+                pass
+            elif step.source == 'fragment' and type(subject) is CaperClosure:
+                Logr.debug('Closure encountered on fragment step, jumping into fragments')
+                return [CaperClosureNode(subject, parent_head, None)], None
+
+            match = step.execute(subject)
+
+            if match.success:
+                if type(match.result) is dict:
+                    match.result = clean_dict(match.result)
+
+                Logr.debug('Found match with weight %s, match: %s, num_fragments: %s' % (
+                    match.weight, match.result, match.num_fragments
+                ))
+
+                step.matched = True
+
+                break
+
+        if all([step.single and step.matched for step in self.steps]):
+            Logr.debug('All steps completed, group finished')
+            parent_node.finished_groups.append(self)
+            return nodes, match
+
+        # Check post constraints
+        broke, definite = self.check_constraints(self.post_constraints, parent_head, subject, match=match)
+        if broke:
+            return nodes, None
+
+        return nodes, match
+
+    def check_constraints(self, constraints, parent_head, subject, **kwargs):
+        parent_node = parent_head[0] if type(parent_head) is list else parent_head
+
+        # Check constraints
+        for constraint in [c for c in constraints if c.target == subject.__key__ or not c.target]:
+            Logr.debug("Testing constraint %s against subject %s", repr(constraint), repr(subject))
+
+            weight, success = constraint.execute(parent_node, subject, **kwargs)
+
+            if success:
+                Logr.debug('capturing broke on "%s" at %s', subject.value, constraint)
+                parent_node.finished_groups.append(self)
+
+                return True, weight == 1.0
+
+        return False, None
+
     def execute(self):
         heads_finished = None
 
@@ -126,20 +254,26 @@ class CaptureGroup(object):
             for head in heads:
                 node = head[0] if type(head) is list else head
 
-                Logr.debug("head node: %s" % node)
-
                 if self in node.finished_groups:
                     Logr.debug("head finished for group")
                     self.result.heads.append(head)
                     heads_finished.append(True)
                     continue
 
+                Logr.debug('')
+
+                Logr.debug(node)
+
                 next_subject = node.next()
 
+                Logr.debug('----------[%s] (%s)----------' % (next_subject, repr(next_subject.value) if next_subject else None))
+
                 if next_subject:
                     for node_result in self.parse_subject(head, next_subject):
                         self.result.heads.append(node_result)
 
+                    Logr.debug('Heads: %s', self.result.heads)
+
                 heads_finished.append(self in node.finished_groups or next_subject is None)
 
             if len(self.result.heads) == 0:

libs/caper/helpers.py

@@ -74,3 +74,7 @@ def xrange_six(start, stop=None, step=None):
             return range(start)
         else:
             return xrange(start)
+
+
+def delta_seconds(td):
+    return (td.microseconds + (td.seconds + td.days * 24 * 3600) * 1e6) / 1e6

libs/caper/matcher.py

@@ -12,9 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import re
+from caper.helpers import is_list_type, update_dict, delta_seconds
+from datetime import datetime
 from logr import Logr
-from caper.helpers import is_list_type, update_dict
+import re
 
 
 class FragmentMatcher(object):
@@ -24,6 +25,9 @@ class FragmentMatcher(object):
         self.construct_patterns(pattern_groups)
 
     def construct_patterns(self, pattern_groups):
+        compile_start = datetime.now()
+        compile_count = 0
+
         for group_name, patterns in pattern_groups:
             if group_name not in self.regex:
                 self.regex[group_name] = []
@@ -54,17 +58,20 @@ class FragmentMatcher(object):
                                 value = value[0]
 
                         result.append(re.compile(value, re.IGNORECASE))
+                        compile_count += 1
 
                     weight_patterns.append(tuple(result))
 
                 self.regex[group_name].append((weight, weight_patterns))
 
+        Logr.info("Compiled %s patterns in %ss", compile_count, delta_seconds(datetime.now() - compile_start))
+
     def find_group(self, name):
         for group_name, weight_groups in self.regex.items():
             if group_name and group_name == name:
                 return group_name, weight_groups
 
-        return None
+        return None, None
 
     def value_match(self, value, group_name=None, single=True):
         result = None

libs/caper/objects.py

@@ -16,7 +16,12 @@ from caper.helpers import xrange_six
 
 
 class CaperClosure(object):
-    def __init__(self, value):
+    __key__ = 'closure'
+
+    def __init__(self, index, value):
+        #: :type: int
+        self.index = index
+
         #: :type: str
         self.value = value
 
@@ -28,9 +33,20 @@ class CaperClosure(object):
         #: :type: list of CaperFragment
         self.fragments = []
 
+    def __str__(self):
+        return "<CaperClosure value: %s" % repr(self.value)
+
+    def __repr__(self):
+        return self.__str__()
+
 
 class CaperFragment(object):
-    def __init__(self):
+    __key__ = 'fragment'
+
+    def __init__(self, closure=None):
+        #: :type: CaperClosure
+        self.closure = closure
+
         #: :type: str
         self.value = ""
 
@@ -73,3 +89,36 @@ class CaperFragment(object):
 
     def take_right(self, count, include_self=True):
         return self.take('right', count, include_self)
+
+    def __str__(self):
+        return "<CaperFragment value: %s" % repr(self.value)
+
+    def __repr__(self):
+        return self.__str__()
+
+
+class CaptureMatch(object):
+    def __init__(self, tag, step, success=False, weight=None, result=None, num_fragments=1):
+        #: :type: bool
+        self.success = success
+
+        #: :type: float
+        self.weight = weight
+
+        #: :type: dict or str
+        self.result = result
+
+        #: :type: int
+        self.num_fragments = num_fragments
+
+        #: :type: str
+        self.tag = tag
+
+        #: :type: CaptureStep
+        self.step = step
+
+    def __str__(self):
+        return "<CaperMatch result: %s>" % repr(self.result)
+
+    def __repr__(self):
+        return self.__str__()

libs/caper/parsers/anime.py

@@ -75,8 +75,8 @@ class AnimeParser(Parser):
             .execute(once=True)
 
         self.capture_fragment('show_name', single=False)\
-            .until(value__re='identifier')\
+            .until_fragment(value__re='identifier')\
-            .until(value__re='video')\
+            .until_fragment(value__re='video')\
             .execute()
 
         self.capture_fragment('identifier', regex='identifier') \

libs/caper/parsers/base.py

@@ -14,14 +14,15 @@
 
 from caper import FragmentMatcher
 from caper.group import CaptureGroup
-from caper.result import CaperResult, CaperClosureNode
+from caper.result import CaperResult, CaperClosureNode, CaperRootNode
+from logr import Logr
 
 
 class Parser(object):
-    def __init__(self, pattern_groups, debug=False):
+    def __init__(self, matcher, debug=False):
         self.debug = debug
 
-        self.matcher = FragmentMatcher(pattern_groups)
+        self.matcher = matcher
 
         self.closures = None
         #: :type: caper.result.CaperResult
@@ -51,7 +52,7 @@ class Parser(object):
         self.reset()
         self.closures = closures
 
-        self.result.heads = [CaperClosureNode(closures[0])]
+        self.result.heads = [CaperRootNode(closures[0])]
 
     def run(self, closures):
         """
@@ -64,18 +65,20 @@ class Parser(object):
     # Capture Methods
     #
 
-    def capture_fragment(self, tag, regex=None, func=None, single=True):
+    def capture_fragment(self, tag, regex=None, func=None, single=True, **kwargs):
         return CaptureGroup(self, self.result).capture_fragment(
             tag,
             regex=regex,
             func=func,
-            single=single
+            single=single,
+            **kwargs
         )
 
-    def capture_closure(self, tag, regex=None, func=None, single=True):
+    def capture_closure(self, tag, regex=None, func=None, single=True, **kwargs):
         return CaptureGroup(self, self.result).capture_closure(
             tag,
             regex=regex,
             func=func,
-            single=single
+            single=single,
+            **kwargs
         )

libs/caper/parsers/scene.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 from logr import Logr
+from caper import FragmentMatcher
 from caper.parsers.base import Parser
 from caper.result import CaperFragmentNode
 
@@ -22,8 +23,10 @@ PATTERN_GROUPS = [
         (1.0, [
             # S01E01-E02
             ('^S(?P<season>\d+)E(?P<episode_from>\d+)$', '^E(?P<episode_to>\d+)$'),
-            # S03 E01 to E08
+            # 'S03 E01 to E08' or 'S03 E01 - E09'
-            ('^S(?P<season>\d+)$', '^E(?P<episode_from>\d+)$', '^to$', '^E(?P<episode_to>\d+)$'),
+            ('^S(?P<season>\d+)$', '^E(?P<episode_from>\d+)$', '^(to|-)$', '^E(?P<episode_to>\d+)$'),
+            # 'E01 to E08' or 'E01 - E09'
+            ('^E(?P<episode_from>\d+)$', '^(to|-)$', '^E(?P<episode_to>\d+)$'),
 
             # S01-S03
             ('^S(?P<season_from>\d+)$', '^S(?P<season_to>\d+)$'),
@@ -58,6 +61,9 @@ PATTERN_GROUPS = [
             # Part.3
             # Part.1.and.Part.3
             ('^Part$', '(?P<part>\d+)'),
+
+            r'(?P<extra>Special)',
+            r'(?P<country>NZ|AU|US|UK)'
         ]),
         (0.8, [
             # 100 - 1899, 2100 - 9999 (skips 1900 to 2099 - so we don't get years my mistake)
@@ -69,6 +75,7 @@ PATTERN_GROUPS = [
             r'^(?P<season>([1-9])|([1-9][0-9]))(?P<episode>\d{2})$'
         ])
     ]),
+
     ('video', [
         r'(?P<aspect>FS|WS)',
 
@@ -152,14 +159,23 @@ PATTERN_GROUPS = [
 
 
 class SceneParser(Parser):
+    matcher = None
+
     def __init__(self, debug=False):
-        super(SceneParser, self).__init__(PATTERN_GROUPS, debug)
+        if not SceneParser.matcher:
+            SceneParser.matcher = FragmentMatcher(PATTERN_GROUPS)
+            Logr.info("Fragment matcher for %s created", self.__class__.__name__)
+
+        super(SceneParser, self).__init__(SceneParser.matcher, debug)
 
     def capture_group(self, fragment):
-        if fragment.left_sep == '-' and not fragment.right:
+        if fragment.closure.index + 1 != len(self.closures):
-            return fragment.value
+            return None
 
-        return None
+        if fragment.left_sep != '-' or fragment.right:
+            return None
+
+        return fragment.value
 
     def run(self, closures):
         """
@@ -169,19 +185,19 @@ class SceneParser(Parser):
         self.setup(closures)
 
         self.capture_fragment('show_name', single=False)\
-            .until(fragment__re='identifier')\
+            .until_fragment(node__re='identifier')\
-            .until(fragment__re='video') \
+            .until_fragment(node__re='video')\
-            .until(fragment__re='dvd') \
+            .until_fragment(node__re='dvd')\
-            .until(fragment__re='audio') \
+            .until_fragment(node__re='audio')\
-            .until(fragment__re='scene') \
+            .until_fragment(node__re='scene')\
             .execute()
 
         self.capture_fragment('identifier', regex='identifier', single=False)\
-            .capture_fragment('video', regex='video', single=False) \
+            .capture_fragment('video', regex='video', single=False)\
-            .capture_fragment('dvd', regex='dvd', single=False) \
+            .capture_fragment('dvd', regex='dvd', single=False)\
-            .capture_fragment('audio', regex='audio', single=False) \
+            .capture_fragment('audio', regex='audio', single=False)\
-            .capture_fragment('scene', regex='scene', single=False) \
+            .capture_fragment('scene', regex='scene', single=False)\
-            .until(left_sep__eq='-', right__eq=None)\
+            .until_fragment(left_sep__eq='-', right__eq=None)\
             .execute()
 
         self.capture_fragment('group', func=self.capture_group)\
@@ -206,7 +222,9 @@ class SceneParser(Parser):
                 Logr.debug(head[0].closure.value)
 
             for node in head:
-                Logr.debug('\t' + str(node).ljust(55) + '\t' + str(node.weight) + '\t' + str(node.match))
+                Logr.debug('\t' + str(node).ljust(55) + '\t' + (
+                    str(node.match.weight) + '\t' + str(node.match.result)
+                ) if node.match else '')
 
             if len(head) > 0 and head[0].parent:
                 self.print_tree([head[0].parent])

libs/caper/parsers/usenet.py

@@ -0,0 +1,115 @@
+# Copyright 2013 Dean Gardiner <gardiner91@gmail.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from logr import Logr
+from caper import FragmentMatcher
+from caper.parsers.base import Parser
+
+
+PATTERN_GROUPS = [
+    ('usenet', [
+        r'\[(?P<group>#[\w\.@]+)\]',
+        r'^\[(?P<code>\w+)\]$',
+        r'\[(?P<full>FULL)\]',
+        r'\[\s?(?P<group>TOWN)\s?\]',
+        r'(.*?\s)?[_\W]*(?P<site>www\..*?\.[a-z0-9]+)[_\W]*(.*?\s)?',
+        r'(.*?\s)?[_\W]*(?P<site>(www\.)?[-\w]+\.(com|org|info))[_\W]*(.*?\s)?'
+    ]),
+
+    ('part', [
+        r'.?(?P<current>\d+)/(?P<total>\d+).?'
+    ]),
+
+    ('detail', [
+        r'[\s-]*\w*?[\s-]*\"(?P<file_name>.*?)\"[\s-]*\w*?[\s-]*(?P<size>[\d,\.]*\s?MB)?[\s-]*(?P<extra>yEnc)?',
+        r'(?P<size>[\d,\.]*\s?MB)[\s-]*(?P<extra>yEnc)',
+        r'(?P<size>[\d,\.]*\s?MB)|(?P<extra>yEnc)'
+    ])
+]
+
+
+class UsenetParser(Parser):
+    matcher = None
+
+    def __init__(self, debug=False):
+        if not UsenetParser.matcher:
+            UsenetParser.matcher = FragmentMatcher(PATTERN_GROUPS)
+            Logr.info("Fragment matcher for %s created", self.__class__.__name__)
+
+        super(UsenetParser, self).__init__(UsenetParser.matcher, debug)
+
+    def run(self, closures):
+        """
+        :type closures: list of CaperClosure
+        """
+
+        self.setup(closures)
+
+        # Capture usenet or part info until we get a part or matching fails
+        self.capture_closure('usenet', regex='usenet', single=False)\
+            .capture_closure('part', regex='part', single=True) \
+            .until_result(tag='part') \
+            .until_failure()\
+            .execute()
+
+        is_town_release, has_part = self.get_state()
+
+        if not is_town_release:
+            self.capture_release_name()
+
+        # If we already have the part (TOWN releases), ignore matching part again
+        if not is_town_release and not has_part:
+            self.capture_fragment('part', regex='part', single=True)\
+                .until_closure(node__re='usenet')\
+                .until_success()\
+                .execute()
+
+        # Capture any leftover details
+        self.capture_closure('usenet', regex='usenet', single=False)\
+            .capture_closure('detail', regex='detail', single=False)\
+            .execute()
+
+        self.result.build()
+        return self.result
+
+    def capture_release_name(self):
+        self.capture_closure('detail', regex='detail', single=False)\
+            .until_failure()\
+            .execute()
+
+        self.capture_fragment('release_name', single=False, include_separators=True) \
+            .until_closure(node__re='usenet') \
+            .until_closure(node__re='detail') \
+            .until_closure(node__re='part') \
+            .until_fragment(value__eq='-')\
+            .execute()
+
+        # Capture any detail after the release name
+        self.capture_closure('detail', regex='detail', single=False)\
+            .until_failure()\
+            .execute()
+
+    def get_state(self):
+        # TODO multiple-chains?
+        is_town_release = False
+        has_part = False
+
+        for tag, result in self.result.heads[0].captured():
+            if tag == 'usenet' and result.get('group') == 'TOWN':
+                is_town_release = True
+
+            if tag == 'part':
+                has_part = True
+
+        return is_town_release, has_part

libs/caper/result.py

@@ -20,7 +20,7 @@ GROUP_MATCHES = ['identifier']
 
 
 class CaperNode(object):
-    def __init__(self, closure, parent=None, tag=None, weight=None, match=None):
+    def __init__(self, closure, parent=None, match=None):
         """
         :type parent: CaperNode
         :type weight: float
@@ -28,41 +28,77 @@ class CaperNode(object):
 
         #: :type: caper.objects.CaperClosure
         self.closure = closure
+
         #: :type: CaperNode
         self.parent = parent
-        #: :type: str
+
-        self.tag = tag
+        #: :type: CaptureMatch
-        #: :type: float
-        self.weight = weight
-        #: :type: dict
         self.match = match
+
         #: :type: list of CaptureGroup
         self.finished_groups = []
 
     def next(self):
         raise NotImplementedError()
 
+    def captured(self):
+        cur = self
 
-class CaperClosureNode(CaperNode):
+        if cur.match:
-    def __init__(self, closure, parent=None, tag=None, weight=None, match=None):
+            yield cur.match.tag, cur.match.result
+
+        while cur.parent:
+            cur = cur.parent
+
+            if cur.match:
+                yield cur.match.tag, cur.match.result
+
+
+class CaperRootNode(CaperNode):
+    def __init__(self, closure):
         """
         :type closure: caper.objects.CaperClosure or list of caper.objects.CaperClosure
         """
-        super(CaperClosureNode, self).__init__(closure, parent, tag, weight, match)
+        super(CaperRootNode, self).__init__(closure)
 
     def next(self):
-        if self.closure and len(self.closure.fragments) > 0:
+        return self.closure
+
+
+class CaperClosureNode(CaperNode):
+    def __init__(self, closure, parent=None, match=None):
+        """
+        :type closure: caper.objects.CaperClosure or list of caper.objects.CaperClosure
+        """
+        super(CaperClosureNode, self).__init__(closure, parent, match)
+
+    def next(self):
+        if not self.closure:
+            return None
+
+        if self.match:
+            # Jump to next closure if we have a match
+            return self.closure.right
+        elif len(self.closure.fragments) > 0:
+            # Otherwise parse the fragments
             return self.closure.fragments[0]
+
         return None
 
+    def __str__(self):
+        return "<CaperClosureNode match: %s>" % repr(self.match)
+
+    def __repr__(self):
+        return self.__str__()
+
 
 class CaperFragmentNode(CaperNode):
-    def __init__(self, closure, fragments, parent=None, tag=None, weight=None, match=None):
+    def __init__(self, closure, fragments, parent=None, match=None):
         """
         :type closure: caper.objects.CaperClosure
         :type fragments: list of caper.objects.CaperFragment
         """
-        super(CaperFragmentNode, self).__init__(closure, parent, tag, weight, match)
+        super(CaperFragmentNode, self).__init__(closure, parent, match)
 
         #: :type: caper.objects.CaperFragment or list of caper.objects.CaperFragment
         self.fragments = fragments
@@ -76,6 +112,12 @@ class CaperFragmentNode(CaperNode):
 
         return None
 
+    def __str__(self):
+        return "<CaperFragmentNode match: %s>" % repr(self.match)
+
+    def __repr__(self):
+        return self.__str__()
+
 
 class CaperResult(object):
     def __init__(self):
@@ -122,15 +164,8 @@ class CaperResult(object):
                 result.append(node_chain)
                 continue
 
-            # Skip over closure nodes
+            node_chain.update(node)
-            if type(node) is CaperClosureNode:
+            result.extend(self.combine_chain(node.parent, node_chain))
-                result.extend(self.combine_chain(node.parent, node_chain))
-
-            # Parse fragment matches
-            if type(node) is CaperFragmentNode:
-                node_chain.update(node)
-
-                result.extend(self.combine_chain(node.parent, node_chain))
 
         return result
 
@@ -145,17 +180,23 @@ class CaperResultChain(object):
         self.weights = []
 
     def update(self, subject):
-        if subject.weight is None:
+        """
+        :type subject: CaperFragmentNode
+        """
+        if not subject.match or not subject.match.success:
             return
 
-        self.num_matched += len(subject.fragments) if subject.fragments is not None else 0
+        # TODO this should support closure nodes
-        self.weights.append(subject.weight)
+        if type(subject) is CaperFragmentNode:
+            self.num_matched += len(subject.fragments) if subject.fragments is not None else 0
+
+        self.weights.append(subject.match.weight)
 
         if subject.match:
-            if subject.tag not in self.info:
+            if subject.match.tag not in self.info:
-                self.info[subject.tag] = []
+                self.info[subject.match.tag] = []
 
-            self.info[subject.tag].insert(0, subject.match)
+            self.info[subject.match.tag].insert(0, subject.match.result)
 
     def finish(self):
         self.weight = sum(self.weights) / len(self.weights)

libs/caper/step.py

@@ -12,13 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from caper.objects import CaptureMatch
 from logr import Logr
 
 
 class CaptureStep(object):
     REPR_KEYS = ['regex', 'func', 'single']
 
-    def __init__(self, capture_group, tag, source, regex=None, func=None, single=None):
+    def __init__(self, capture_group, tag, source, regex=None, func=None, single=None, **kwargs):
         #: @type: CaptureGroup
         self.capture_group = capture_group
 
@@ -33,22 +34,57 @@ class CaptureStep(object):
         #: @type: bool
         self.single = single
 
+        self.kwargs = kwargs
+
+        self.matched = False
+
     def execute(self, fragment):
+        """Execute step on fragment
+
+        :type fragment: CaperFragment
+        :rtype : CaptureMatch
+        """
+
+        match = CaptureMatch(self.tag, self)
+
         if self.regex:
-            weight, match, num_fragments = self.capture_group.parser.matcher.fragment_match(fragment, self.regex)
+            weight, result, num_fragments = self.capture_group.parser.matcher.fragment_match(fragment, self.regex)
             Logr.debug('(execute) [regex] tag: "%s"', self.tag)
-            if match:
+
-                return True, weight, match, num_fragments
+            if not result:
+                return match
+
+            # Populate CaptureMatch
+            match.success = True
+            match.weight = weight
+            match.result = result
+            match.num_fragments = num_fragments
         elif self.func:
-            match = self.func(fragment)
+            result = self.func(fragment)
             Logr.debug('(execute) [func] %s += "%s"', self.tag, match)
-            if match:
+
-                return True, 1.0, match, 1
+            if not result:
+                return match
+
+            # Populate CaptureMatch
+            match.success = True
+            match.weight = 1.0
+            match.result = result
         else:
             Logr.debug('(execute) [raw] %s += "%s"', self.tag, fragment.value)
-            return True, 1.0, fragment.value, 1
 
-        return False, None, None, 1
+            include_separators = self.kwargs.get('include_separators', False)
+
+            # Populate CaptureMatch
+            match.success = True
+            match.weight = 1.0
+
+            if include_separators:
+                match.result = (fragment.left_sep, fragment.value, fragment.right_sep)
+            else:
+                match.result = fragment.value
+
+        return match
 
     def __repr__(self):
         attribute_values = [key + '=' + repr(getattr(self, key))