From f648af66a62567fa7eef53c70fc841a8207fdfe0 Mon Sep 17 00:00:00 2001 From: Dean Gardiner Date: Sun, 8 Dec 2013 22:40:39 +1300 Subject: [PATCH 1/6] Moved matcher plugin to core/media, moved some matcher related functions from ShowSearcher to ShowMatcher --- .../_base}/matcher/__init__.py | 0 couchpotato/core/media/_base/matcher/base.py | 48 ++++++ couchpotato/core/media/_base/matcher/main.py | 85 ++++++++++ .../core/media/show/matcher/__init__.py | 6 + couchpotato/core/media/show/matcher/main.py | 86 ++++++++++ couchpotato/core/media/show/searcher/main.py | 41 +---- couchpotato/core/plugins/matcher/main.py | 157 ------------------ 7 files changed, 226 insertions(+), 197 deletions(-) rename couchpotato/core/{plugins => media/_base}/matcher/__init__.py (100%) create mode 100644 couchpotato/core/media/_base/matcher/base.py create mode 100644 couchpotato/core/media/_base/matcher/main.py create mode 100644 couchpotato/core/media/show/matcher/__init__.py create mode 100644 couchpotato/core/media/show/matcher/main.py delete mode 100644 couchpotato/core/plugins/matcher/main.py diff --git a/couchpotato/core/plugins/matcher/__init__.py b/couchpotato/core/media/_base/matcher/__init__.py similarity index 100% rename from couchpotato/core/plugins/matcher/__init__.py rename to couchpotato/core/media/_base/matcher/__init__.py diff --git a/couchpotato/core/media/_base/matcher/base.py b/couchpotato/core/media/_base/matcher/base.py new file mode 100644 index 00000000..399d1fe1 --- /dev/null +++ b/couchpotato/core/media/_base/matcher/base.py @@ -0,0 +1,48 @@ +from couchpotato.core.helpers.encoding import simplifyString +from couchpotato.core.logger import CPLog +from couchpotato.core.plugins.base import Plugin + +log = CPLog(__name__) + + +class MatcherBase(Plugin): + def flattenInfo(self, info): + flat_info = {} + + for match in info: + for key, value in match.items(): + if key not in flat_info: + flat_info[key] = [] + + flat_info[key].append(value) + + return flat_info + + def simplifyValue(self, value): + if not value: + return value + + if isinstance(value, basestring): + return simplifyString(value) + + if isinstance(value, list): + return [self.simplifyValue(x) for x in value] + + raise ValueError("Unsupported value type") + + def chainMatch(self, chain, group, tags): + info = self.flattenInfo(chain.info[group]) + + found_tags = [] + for tag, accepted in tags.items(): + values = [self.simplifyValue(x) for x in info.get(tag, [None])] + + if any([val in accepted for val in values]): + found_tags.append(tag) + + log.debug('tags found: %s, required: %s' % (found_tags, tags.keys())) + + if set(tags.keys()) == set(found_tags): + return True + + return all([key in found_tags for key, value in tags.items()]) diff --git a/couchpotato/core/media/_base/matcher/main.py b/couchpotato/core/media/_base/matcher/main.py new file mode 100644 index 00000000..9cc8c1a0 --- /dev/null +++ b/couchpotato/core/media/_base/matcher/main.py @@ -0,0 +1,85 @@ +from couchpotato.core.event import addEvent, fireEvent +from couchpotato.core.helpers.variable import possibleTitles +from couchpotato.core.logger import CPLog +from couchpotato.core.media._base.matcher.base import MatcherBase +from caper import Caper + +log = CPLog(__name__) + + +class Matcher(MatcherBase): + def __init__(self): + super(Matcher, self).__init__() + + self.caper = Caper() + + addEvent('matcher.parse', self.parse) + addEvent('matcher.match', self.match) + + addEvent('matcher.correct_title', self.correctTitle) + addEvent('matcher.correct_quality', self.correctQuality) + + def parse(self, name, parser='scene'): + return self.caper.parse(name, parser) + + def match(self, release, media, quality): + match = fireEvent('matcher.parse', release['name'], single = True) + + if len(match.chains) < 1: + log.info2('Wrong: %s, unable to parse release name (no chains)', release['name']) + return False + + for chain in match.chains: + if fireEvent('%s.matcher.correct' % media['type'], chain, release, media, quality, single = True): + return chain + + return False + + def correctTitle(self, chain, media): + root_library = media['library']['root_library'] + + if 'show_name' not in chain.info or not len(chain.info['show_name']): + log.info('Wrong: missing show name in parsed result') + return False + + # Get the lower-case parsed show name from the chain + chain_words = [x.lower() for x in chain.info['show_name']] + + # Build a list of possible titles of the media we are searching for + titles = root_library['info']['titles'] + + # Add year suffix titles (will result in ['', ' ', '', ...]) + suffixes = [None, root_library['info']['year']] + + titles = [ + title + ((' %s' % suffix) if suffix else '') + for title in titles + for suffix in suffixes + ] + + # Check show titles match + # TODO check xem names + for title in titles: + for valid_words in [x.split(' ') for x in possibleTitles(title)]: + + if valid_words == chain_words: + return True + + return False + + def correctQuality(self, chain, quality, quality_map): + if quality['identifier'] not in quality_map: + log.info2('Wrong: unknown preferred quality %s', quality['identifier']) + return False + + if 'video' not in chain.info: + log.info2('Wrong: no video tags found') + return False + + video_tags = quality_map[quality['identifier']] + + if not self.chainMatch(chain, 'video', video_tags): + log.info2('Wrong: %s tags not in chain', video_tags) + return False + + return True diff --git a/couchpotato/core/media/show/matcher/__init__.py b/couchpotato/core/media/show/matcher/__init__.py new file mode 100644 index 00000000..489ef675 --- /dev/null +++ b/couchpotato/core/media/show/matcher/__init__.py @@ -0,0 +1,6 @@ +from .main import ShowMatcher + +def start(): + return ShowMatcher() + +config = [] diff --git a/couchpotato/core/media/show/matcher/main.py b/couchpotato/core/media/show/matcher/main.py new file mode 100644 index 00000000..c9a54532 --- /dev/null +++ b/couchpotato/core/media/show/matcher/main.py @@ -0,0 +1,86 @@ +from couchpotato import CPLog +from couchpotato.core.event import addEvent, fireEvent +from couchpotato.core.helpers.variable import dictIsSubset, tryInt, toIterable +from couchpotato.core.media._base.matcher.base import MatcherBase + +log = CPLog(__name__) + + +class ShowMatcher(MatcherBase): + + type = ['show', 'season', 'episode'] + + # TODO come back to this later, think this could be handled better, this is starting to get out of hand.... + quality_map = { + 'bluray_1080p': {'resolution': ['1080p'], 'source': ['bluray']}, + 'bluray_720p': {'resolution': ['720p'], 'source': ['bluray']}, + + 'bdrip_1080p': {'resolution': ['1080p'], 'source': ['BDRip']}, + 'bdrip_720p': {'resolution': ['720p'], 'source': ['BDRip']}, + + 'brrip_1080p': {'resolution': ['1080p'], 'source': ['BRRip']}, + 'brrip_720p': {'resolution': ['720p'], 'source': ['BRRip']}, + + 'webdl_1080p': {'resolution': ['1080p'], 'source': ['webdl', ['web', 'dl']]}, + 'webdl_720p': {'resolution': ['720p'], 'source': ['webdl', ['web', 'dl']]}, + 'webdl_480p': {'resolution': ['480p'], 'source': ['webdl', ['web', 'dl']]}, + + 'hdtv_720p': {'resolution': ['720p'], 'source': ['hdtv']}, + 'hdtv_sd': {'resolution': ['480p', None], 'source': ['hdtv']}, + } + + def __init__(self): + super(ShowMatcher, self).__init__() + + for type in toIterable(self.type): + addEvent('%s.matcher.correct' % type, self.correct) + addEvent('%s.matcher.correct_identifier' % type, self.correctIdentifier) + + def correct(self, chain, release, media, quality): + log.info("Checking if '%s' is valid", release['name']) + log.info2('Release parsed as: %s', chain.info) + + if not fireEvent('matcher.correct_quality', chain, quality, self.quality_map, single = True): + log.info('Wrong: %s, quality does not match', release['name']) + return False + + if not fireEvent('show.matcher.correct_identifier', chain, media): + log.info('Wrong: %s, identifier does not match', release['name']) + return False + + if not fireEvent('matcher.correct_title', chain, media): + log.info("Wrong: '%s', undetermined naming.", (' '.join(chain.info['show_name']))) + return False + + return True + + def correctIdentifier(self, chain, media): + required_id = fireEvent('library.identifier', media['library'], single = True) + + if 'identifier' not in chain.info: + return False + + # TODO could be handled better? + if len(chain.info['identifier']) != 1: + return False + identifier = chain.info['identifier'][0] + + # TODO air by date episodes + + # TODO this should support identifiers with characters 'a', 'b', etc.. + for k, v in identifier.items(): + identifier[k] = tryInt(v, None) + + if any([x in identifier for x in ['episode_from', 'episode_to']]): + log.info2('Wrong: releases with identifier ranges are not supported yet') + return False + + # 'episode' is required in identifier for subset matching + if 'episode' not in identifier: + identifier['episode'] = None + + if not dictIsSubset(required_id, identifier): + log.info2('Wrong: required identifier %s does not match release identifier %s', (str(required_id), str(identifier))) + return False + + return True diff --git a/couchpotato/core/media/show/searcher/main.py b/couchpotato/core/media/show/searcher/main.py index 1ce30341..c91a8556 100644 --- a/couchpotato/core/media/show/searcher/main.py +++ b/couchpotato/core/media/show/searcher/main.py @@ -18,25 +18,6 @@ class ShowSearcher(Plugin): in_progress = False - # TODO come back to this later, think this could be handled better, this is starting to get out of hand.... - quality_map = { - 'bluray_1080p': {'resolution': ['1080p'], 'source': ['bluray']}, - 'bluray_720p': {'resolution': ['720p'], 'source': ['bluray']}, - - 'bdrip_1080p': {'resolution': ['1080p'], 'source': ['BDRip']}, - 'bdrip_720p': {'resolution': ['720p'], 'source': ['BDRip']}, - - 'brrip_1080p': {'resolution': ['1080p'], 'source': ['BRRip']}, - 'brrip_720p': {'resolution': ['720p'], 'source': ['BRRip']}, - - 'webdl_1080p': {'resolution': ['1080p'], 'source': ['webdl', ['web', 'dl']]}, - 'webdl_720p': {'resolution': ['720p'], 'source': ['webdl', ['web', 'dl']]}, - 'webdl_480p': {'resolution': ['480p'], 'source': ['webdl', ['web', 'dl']]}, - - 'hdtv_720p': {'resolution': ['720p'], 'source': ['hdtv']}, - 'hdtv_sd': {'resolution': ['480p', None], 'source': ['hdtv']}, - } - def __init__(self): super(ShowSearcher, self).__init__() @@ -46,8 +27,6 @@ class ShowSearcher(Plugin): addEvent('%s.searcher.single' % type, self.single) addEvent('searcher.get_search_title', self.getSearchTitle) - - addEvent('searcher.correct_match', self.correctMatch) addEvent('searcher.correct_release', self.correctRelease) def single(self, media, search_protocols = None, manual = False): @@ -234,30 +213,12 @@ class ShowSearcher(Plugin): return False # TODO Matching is quite costly, maybe we should be caching release matches somehow? (also look at caper optimizations) - match = fireEvent('matcher.best', release, media, quality, single = True) + match = fireEvent('matcher.match', release, media, quality, single = True) if match: return match.weight return False - def correctMatch(self, chain, release, media, quality): - log.info("Checking if '%s' is valid", release['name']) - log.info2('Release parsed as: %s', chain.info) - - if not fireEvent('matcher.correct_quality', chain, quality, self.quality_map, single = True): - log.info('Wrong: %s, quality does not match', release['name']) - return False - - if not fireEvent('matcher.correct_identifier', chain, media): - log.info('Wrong: %s, identifier does not match', release['name']) - return False - - if not fireEvent('matcher.correct_title', chain, media): - log.info("Wrong: '%s', undetermined naming.", (' '.join(chain.info['show_name']))) - return False - - return True - def getLibraries(self, library): if 'related_libraries' not in library: log.warning("'related_libraries' missing from media library, unable to continue searching") diff --git a/couchpotato/core/plugins/matcher/main.py b/couchpotato/core/plugins/matcher/main.py deleted file mode 100644 index fb3bfc11..00000000 --- a/couchpotato/core/plugins/matcher/main.py +++ /dev/null @@ -1,157 +0,0 @@ -from caper import Caper -from couchpotato import CPLog, tryInt -from couchpotato.core.event import addEvent, fireEvent -from couchpotato.core.helpers.encoding import simplifyString -from couchpotato.core.helpers.variable import possibleTitles, dictIsSubset -from couchpotato.core.plugins.base import Plugin - -log = CPLog(__name__) - - -class Matcher(Plugin): - def __init__(self): - self.caper = Caper() - - addEvent('matcher.parse', self.parse) - addEvent('matcher.best', self.best) - - addEvent('matcher.correct_title', self.correctTitle) - addEvent('matcher.correct_identifier', self.correctIdentifier) - addEvent('matcher.correct_quality', self.correctQuality) - - def parse(self, release): - return self.caper.parse(release['name']) - - def best(self, release, media, quality): - match = fireEvent('matcher.parse', release, single = True) - - if len(match.chains) < 1: - log.info2('Wrong: %s, unable to parse release name (no chains)', release['name']) - return False - - for chain in match.chains: - if fireEvent('searcher.correct_match', chain, release, media, quality, single = True): - return chain - - return False - - def flattenInfo(self, info): - flat_info = {} - - for match in info: - for key, value in match.items(): - if key not in flat_info: - flat_info[key] = [] - - flat_info[key].append(value) - - return flat_info - - def simplifyValue(self, value): - if not value: - return value - - if isinstance(value, basestring): - return simplifyString(value) - - if isinstance(value, list): - return [self.simplifyValue(x) for x in value] - - raise ValueError("Unsupported value type") - - def chainMatch(self, chain, group, tags): - info = self.flattenInfo(chain.info[group]) - - found_tags = [] - for tag, accepted in tags.items(): - values = [self.simplifyValue(x) for x in info.get(tag, [None])] - - if any([val in accepted for val in values]): - found_tags.append(tag) - - log.debug('tags found: %s, required: %s' % (found_tags, tags.keys())) - - if set(tags.keys()) == set(found_tags): - return True - - return all([key in found_tags for key, value in tags.items()]) - - def correctIdentifier(self, chain, media): - required_id = fireEvent('library.identifier', media['library'], single = True) - - if 'identifier' not in chain.info: - return False - - # TODO could be handled better? - if len(chain.info['identifier']) != 1: - return False - identifier = chain.info['identifier'][0] - - # TODO air by date episodes - - # TODO this should support identifiers with characters 'a', 'b', etc.. - for k, v in identifier.items(): - identifier[k] = tryInt(v, None) - - if any([x in identifier for x in ['episode_from', 'episode_to']]): - log.info2('Wrong: releases with identifier ranges are not supported yet') - return False - - # 'episode' is required in identifier for subset matching - if 'episode' not in identifier: - identifier['episode'] = None - - if not dictIsSubset(required_id, identifier): - log.info2('Wrong: required identifier %s does not match release identifier %s', (str(required_id), str(identifier))) - return False - - return True - - def correctTitle(self, chain, media): - root_library = media['library']['root_library'] - - if 'show_name' not in chain.info or not len(chain.info['show_name']): - log.info('Wrong: missing show name in parsed result') - return False - - # Get the lower-case parsed show name from the chain - chain_words = [x.lower() for x in chain.info['show_name']] - - # Build a list of possible titles of the media we are searching for - titles = root_library['info']['titles'] - - # Add year suffix titles (will result in ['', ' ', '', ...]) - suffixes = [None, root_library['info']['year']] - - titles = [ - title + ((' %s' % suffix) if suffix else '') - for title in titles - for suffix in suffixes - ] - - # Check show titles match - # TODO check xem names - for title in titles: - for valid_words in [x.split(' ') for x in possibleTitles(title)]: - - if valid_words == chain_words: - return True - - return False - - def correctQuality(self, chain, quality, quality_map): - if quality['identifier'] not in quality_map: - log.info2('Wrong: unknown preferred quality %s', quality['identifier']) - return False - - if 'video' not in chain.info: - log.info2('Wrong: no video tags found') - return False - - video_tags = quality_map[quality['identifier']] - - if not self.chainMatch(chain, 'video', video_tags): - log.info2('Wrong: %s tags not in chain', video_tags) - return False - - return True From 93aa5b1920000d6058c7e6f830ece5ed95af3118 Mon Sep 17 00:00:00 2001 From: Dean Gardiner Date: Mon, 9 Dec 2013 11:23:48 +1300 Subject: [PATCH 2/6] Updated Caper to v0.2.9 --- libs/caper/__init__.py | 40 ++++++++++++++++++++++++--------- libs/caper/helpers.py | 4 ++++ libs/caper/matcher.py | 11 ++++++++-- libs/caper/objects.py | 10 +++++++-- libs/caper/parsers/base.py | 5 +++-- libs/caper/parsers/scene.py | 44 +++++++++++++++++++++++++------------ 6 files changed, 84 insertions(+), 30 deletions(-) diff --git a/libs/caper/__init__.py b/libs/caper/__init__.py index 9637c066..8b2e61a0 100644 --- a/libs/caper/__init__.py +++ b/libs/caper/__init__.py @@ -19,7 +19,7 @@ from caper.parsers.anime import AnimeParser from caper.parsers.scene import SceneParser -__version_info__ = ('0', '2', '6') +__version_info__ = ('0', '2', '9') __version_branch__ = 'master' __version__ = "%s%s" % ( @@ -44,9 +44,11 @@ CL_END = 1 class Caper(object): def __init__(self, debug=False): + self.debug = debug + self.parsers = { - 'scene': SceneParser(debug), - 'anime': AnimeParser(debug) + 'scene': SceneParser, + 'anime': AnimeParser } def _closure_split(self, name): @@ -63,7 +65,7 @@ class Caper(object): if len(buf) < 1: return - cur = CaperClosure(buf) + cur = CaperClosure(len(closures), buf) cur.left = closures[len(closures) - 1] if len(closures) > 0 else None if cur.left: @@ -109,7 +111,7 @@ class Caper(object): """ cur_position = 0 - cur = CaperFragment() + cur = None def end_fragment(fragments, cur, cur_position): cur.position = cur_position @@ -126,23 +128,41 @@ class Caper(object): for closure in closures: closure.fragments = [] + separator_buffer = "" + for x, ch in enumerate(self._clean_closure(closure.value)): + if not cur: + cur = CaperFragment(closure) + if ch in FRAGMENT_SEPARATORS: - end_fragment(closure.fragments, cur, cur_position) + if cur.value: + separator_buffer = "" + + separator_buffer += ch + + if cur.value or not closure.fragments: + end_fragment(closure.fragments, cur, cur_position) + elif len(separator_buffer) > 1: + cur.value = separator_buffer.strip() + + if cur.value: + end_fragment(closure.fragments, cur, cur_position) + + separator_buffer = "" # Reset - cur = CaperFragment() + cur = None cur_position += 1 else: cur.value += ch # Finish parsing the last fragment - if cur.value != "": + if cur and cur.value: end_fragment(closure.fragments, cur, cur_position) # Reset cur_position = 0 - cur = CaperFragment() + cur = None return closures @@ -158,4 +178,4 @@ class Caper(object): raise ValueError("Unknown parser") # TODO autodetect the parser type - return self.parsers[parser].run(closures) + return self.parsers[parser](self.debug).run(closures) diff --git a/libs/caper/helpers.py b/libs/caper/helpers.py index 2b27e578..ded5d482 100644 --- a/libs/caper/helpers.py +++ b/libs/caper/helpers.py @@ -74,3 +74,7 @@ def xrange_six(start, stop=None, step=None): return range(start) else: return xrange(start) + + +def delta_seconds(td): + return (td.microseconds + (td.seconds + td.days * 24 * 3600) * 1e6) / 1e6 diff --git a/libs/caper/matcher.py b/libs/caper/matcher.py index c71da971..c154cd70 100644 --- a/libs/caper/matcher.py +++ b/libs/caper/matcher.py @@ -12,9 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import re +from caper.helpers import is_list_type, update_dict, delta_seconds +from datetime import datetime from logr import Logr -from caper.helpers import is_list_type, update_dict +import re class FragmentMatcher(object): @@ -24,6 +25,9 @@ class FragmentMatcher(object): self.construct_patterns(pattern_groups) def construct_patterns(self, pattern_groups): + compile_start = datetime.now() + compile_count = 0 + for group_name, patterns in pattern_groups: if group_name not in self.regex: self.regex[group_name] = [] @@ -54,11 +58,14 @@ class FragmentMatcher(object): value = value[0] result.append(re.compile(value, re.IGNORECASE)) + compile_count += 1 weight_patterns.append(tuple(result)) self.regex[group_name].append((weight, weight_patterns)) + Logr.info("Compiled %s patterns in %ss", compile_count, delta_seconds(datetime.now() - compile_start)) + def find_group(self, name): for group_name, weight_groups in self.regex.items(): if group_name and group_name == name: diff --git a/libs/caper/objects.py b/libs/caper/objects.py index 4804deab..1f82c33e 100644 --- a/libs/caper/objects.py +++ b/libs/caper/objects.py @@ -16,7 +16,10 @@ from caper.helpers import xrange_six class CaperClosure(object): - def __init__(self, value): + def __init__(self, index, value): + #: :type: int + self.index = index + #: :type: str self.value = value @@ -30,7 +33,10 @@ class CaperClosure(object): class CaperFragment(object): - def __init__(self): + def __init__(self, closure=None): + #: :type: CaperClosure + self.closure = closure + #: :type: str self.value = "" diff --git a/libs/caper/parsers/base.py b/libs/caper/parsers/base.py index 6f79be61..6bae537c 100644 --- a/libs/caper/parsers/base.py +++ b/libs/caper/parsers/base.py @@ -15,13 +15,14 @@ from caper import FragmentMatcher from caper.group import CaptureGroup from caper.result import CaperResult, CaperClosureNode +from logr import Logr class Parser(object): - def __init__(self, pattern_groups, debug=False): + def __init__(self, matcher, debug=False): self.debug = debug - self.matcher = FragmentMatcher(pattern_groups) + self.matcher = matcher self.closures = None #: :type: caper.result.CaperResult diff --git a/libs/caper/parsers/scene.py b/libs/caper/parsers/scene.py index b96967b9..0dfe378a 100644 --- a/libs/caper/parsers/scene.py +++ b/libs/caper/parsers/scene.py @@ -13,6 +13,7 @@ # limitations under the License. from logr import Logr +from caper import FragmentMatcher from caper.parsers.base import Parser from caper.result import CaperFragmentNode @@ -22,8 +23,10 @@ PATTERN_GROUPS = [ (1.0, [ # S01E01-E02 ('^S(?P\d+)E(?P\d+)$', '^E(?P\d+)$'), - # S03 E01 to E08 - ('^S(?P\d+)$', '^E(?P\d+)$', '^to$', '^E(?P\d+)$'), + # 'S03 E01 to E08' or 'S03 E01 - E09' + ('^S(?P\d+)$', '^E(?P\d+)$', '^(to|-)$', '^E(?P\d+)$'), + # 'E01 to E08' or 'E01 - E09' + ('^E(?P\d+)$', '^(to|-)$', '^E(?P\d+)$'), # S01-S03 ('^S(?P\d+)$', '^S(?P\d+)$'), @@ -58,6 +61,9 @@ PATTERN_GROUPS = [ # Part.3 # Part.1.and.Part.3 ('^Part$', '(?P\d+)'), + + r'(?PSpecial)', + r'(?PNZ|AU|US|UK)' ]), (0.8, [ # 100 - 1899, 2100 - 9999 (skips 1900 to 2099 - so we don't get years my mistake) @@ -69,6 +75,7 @@ PATTERN_GROUPS = [ r'^(?P([1-9])|([1-9][0-9]))(?P\d{2})$' ]) ]), + ('video', [ r'(?PFS|WS)', @@ -152,14 +159,23 @@ PATTERN_GROUPS = [ class SceneParser(Parser): + matcher = None + def __init__(self, debug=False): - super(SceneParser, self).__init__(PATTERN_GROUPS, debug) + if not SceneParser.matcher: + SceneParser.matcher = FragmentMatcher(PATTERN_GROUPS) + Logr.info("Fragment matcher for %s created", self.__class__.__name__) + + super(SceneParser, self).__init__(SceneParser.matcher, debug) def capture_group(self, fragment): - if fragment.left_sep == '-' and not fragment.right: - return fragment.value + if fragment.closure.index + 1 != len(self.closures): + return None - return None + if fragment.left_sep != '-' or fragment.right: + return None + + return fragment.value def run(self, closures): """ @@ -170,17 +186,17 @@ class SceneParser(Parser): self.capture_fragment('show_name', single=False)\ .until(fragment__re='identifier')\ - .until(fragment__re='video') \ - .until(fragment__re='dvd') \ - .until(fragment__re='audio') \ - .until(fragment__re='scene') \ + .until(fragment__re='video')\ + .until(fragment__re='dvd')\ + .until(fragment__re='audio')\ + .until(fragment__re='scene')\ .execute() self.capture_fragment('identifier', regex='identifier', single=False)\ - .capture_fragment('video', regex='video', single=False) \ - .capture_fragment('dvd', regex='dvd', single=False) \ - .capture_fragment('audio', regex='audio', single=False) \ - .capture_fragment('scene', regex='scene', single=False) \ + .capture_fragment('video', regex='video', single=False)\ + .capture_fragment('dvd', regex='dvd', single=False)\ + .capture_fragment('audio', regex='audio', single=False)\ + .capture_fragment('scene', regex='scene', single=False)\ .until(left_sep__eq='-', right__eq=None)\ .execute() From 319c9e979a42dfe1761edc791824da0c498e2717 Mon Sep 17 00:00:00 2001 From: Dean Gardiner Date: Mon, 9 Dec 2013 13:32:43 +1300 Subject: [PATCH 3/6] Split ShowMatcher into Episode and Season matchers, updated correctIdentifier method so there should be less false matches now. --- couchpotato/core/media/_base/matcher/base.py | 10 +++ .../core/media/show/library/season/main.py | 5 +- couchpotato/core/media/show/matcher/main.py | 89 ++++++++++++++----- couchpotato/core/media/show/searcher/main.py | 2 +- 4 files changed, 77 insertions(+), 29 deletions(-) diff --git a/couchpotato/core/media/_base/matcher/base.py b/couchpotato/core/media/_base/matcher/base.py index 399d1fe1..c4b59b25 100644 --- a/couchpotato/core/media/_base/matcher/base.py +++ b/couchpotato/core/media/_base/matcher/base.py @@ -1,3 +1,4 @@ +from couchpotato.core.event import addEvent from couchpotato.core.helpers.encoding import simplifyString from couchpotato.core.logger import CPLog from couchpotato.core.plugins.base import Plugin @@ -6,6 +7,15 @@ log = CPLog(__name__) class MatcherBase(Plugin): + type = None + + def __init__(self): + if self.type: + addEvent('%s.matcher.correct' % self.type, self.correct) + + def correct(self, chain, release, media, quality): + raise NotImplementedError() + def flattenInfo(self, info): flat_info = {} diff --git a/couchpotato/core/media/show/library/season/main.py b/couchpotato/core/media/show/library/season/main.py index 48d201ed..11cf2771 100644 --- a/couchpotato/core/media/show/library/season/main.py +++ b/couchpotato/core/media/show/library/season/main.py @@ -26,11 +26,8 @@ class SeasonLibraryPlugin(LibraryBase): if library.get('type') != 'season': return - season_num = tryInt(library['season_number'], None) - return { - 'season': season_num, - 'episode': None + 'season': tryInt(library['season_number'], None) } def add(self, attrs = {}, update_after = True): diff --git a/couchpotato/core/media/show/matcher/main.py b/couchpotato/core/media/show/matcher/main.py index c9a54532..93515a2f 100644 --- a/couchpotato/core/media/show/matcher/main.py +++ b/couchpotato/core/media/show/matcher/main.py @@ -2,14 +2,18 @@ from couchpotato import CPLog from couchpotato.core.event import addEvent, fireEvent from couchpotato.core.helpers.variable import dictIsSubset, tryInt, toIterable from couchpotato.core.media._base.matcher.base import MatcherBase +from couchpotato.core.providers.base import MultiProvider log = CPLog(__name__) -class ShowMatcher(MatcherBase): +class ShowMatcher(MultiProvider): - type = ['show', 'season', 'episode'] + def getTypes(self): + return [Season, Episode] + +class Base(MatcherBase): # TODO come back to this later, think this could be handled better, this is starting to get out of hand.... quality_map = { 'bluray_1080p': {'resolution': ['1080p'], 'source': ['bluray']}, @@ -30,11 +34,9 @@ class ShowMatcher(MatcherBase): } def __init__(self): - super(ShowMatcher, self).__init__() + super(Base, self).__init__() - for type in toIterable(self.type): - addEvent('%s.matcher.correct' % type, self.correct) - addEvent('%s.matcher.correct_identifier' % type, self.correctIdentifier) + addEvent('%s.matcher.correct_identifier' % self.type, self.correctIdentifier) def correct(self, chain, release, media, quality): log.info("Checking if '%s' is valid", release['name']) @@ -44,7 +46,7 @@ class ShowMatcher(MatcherBase): log.info('Wrong: %s, quality does not match', release['name']) return False - if not fireEvent('show.matcher.correct_identifier', chain, media): + if not fireEvent('%s.matcher.correct_identifier' % self.type, chain, media): log.info('Wrong: %s, identifier does not match', release['name']) return False @@ -55,32 +57,71 @@ class ShowMatcher(MatcherBase): return True def correctIdentifier(self, chain, media): - required_id = fireEvent('library.identifier', media['library'], single = True) + raise NotImplementedError() + def getChainIdentifier(self, chain): if 'identifier' not in chain.info: + return None + + identifier = self.flattenInfo(chain.info['identifier']) + + # Try cast values to integers + for key, value in identifier.items(): + if isinstance(value, list): + if len(value) <= 1: + value = value[0] + else: + log.warning('Wrong: identifier contains multiple season or episode values, unsupported') + return None + + identifier[key] = tryInt(value, value) + + return identifier + + +class Episode(Base): + type = 'episode' + + def correctIdentifier(self, chain, media): + identifier = self.getChainIdentifier(chain) + if not identifier: + log.info2('Wrong: release identifier is not valid (unsupported or missing identifier)') return False - # TODO could be handled better? - if len(chain.info['identifier']) != 1: - return False - identifier = chain.info['identifier'][0] - - # TODO air by date episodes - - # TODO this should support identifiers with characters 'a', 'b', etc.. - for k, v in identifier.items(): - identifier[k] = tryInt(v, None) - + # TODO - Parse episode ranges from identifier to determine if they are multi-part episodes if any([x in identifier for x in ['episode_from', 'episode_to']]): log.info2('Wrong: releases with identifier ranges are not supported yet') return False - # 'episode' is required in identifier for subset matching - if 'episode' not in identifier: - identifier['episode'] = None + required = fireEvent('library.identifier', media['library'], single = True) - if not dictIsSubset(required_id, identifier): - log.info2('Wrong: required identifier %s does not match release identifier %s', (str(required_id), str(identifier))) + # TODO - Support air by date episodes + # TODO - Support episode parts + + if identifier != required: + log.info2('Wrong: required identifier (%s) does not match release identifier (%s)', (required, identifier)) + return False + + return True + +class Season(Base): + type = 'season' + + def correctIdentifier(self, chain, media): + identifier = self.getChainIdentifier(chain) + if not identifier: + log.info2('Wrong: release identifier is not valid (unsupported or missing identifier)') + return False + + # TODO - Parse episode ranges from identifier to determine if they are season packs + if any([x in identifier for x in ['episode_from', 'episode_to']]): + log.info2('Wrong: releases with identifier ranges are not supported yet') + return False + + required = fireEvent('library.identifier', media['library'], single = True) + + if identifier != required: + log.info2('Wrong: required identifier (%s) does not match release identifier (%s)', (required, identifier)) return False return True diff --git a/couchpotato/core/media/show/searcher/main.py b/couchpotato/core/media/show/searcher/main.py index c91a8556..b02143dd 100644 --- a/couchpotato/core/media/show/searcher/main.py +++ b/couchpotato/core/media/show/searcher/main.py @@ -193,7 +193,7 @@ class ShowSearcher(Plugin): if identifier['season']: title += ' S%02d' % identifier['season'] - if identifier['episode']: + if identifier.get('episode'): title += 'E%02d' % identifier['episode'] return title From 2520b19798c567da96328cb7c08124dd82555675 Mon Sep 17 00:00:00 2001 From: Dean Gardiner Date: Mon, 9 Dec 2013 13:33:25 +1300 Subject: [PATCH 4/6] Fixed bug in searcher where episode searches would be triggered if a season release has already been snatched at a better quality --- couchpotato/core/media/show/searcher/main.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/couchpotato/core/media/show/searcher/main.py b/couchpotato/core/media/show/searcher/main.py index b02143dd..f56a0f9f 100644 --- a/couchpotato/core/media/show/searcher/main.py +++ b/couchpotato/core/media/show/searcher/main.py @@ -74,6 +74,8 @@ class ShowSearcher(Plugin): fireEvent('notify.frontend', type = 'show.searcher.started.%s' % media['id'], data = True, message = 'Searching for "%s"' % default_title) ret = False + has_better_quality = None + for quality_type in media['profile']['types']: # TODO check air date? #if not self.conf('always_search') and not self.couldBeReleased(quality_type['quality']['identifier'] in pre_releases, release_dates, movie['library']['year']): @@ -128,7 +130,7 @@ class ShowSearcher(Plugin): if len(too_early_to_search) > 0: log.info2('Too early to search for %s, %s', (too_early_to_search, default_title)) - elif media['type'] == 'season' and not ret: + elif media['type'] == 'season' and not ret and has_better_quality is 0: # If nothing was found, start searching for episodes individually log.info('No season pack found, starting individual episode search') From eb151a4c5d8b02dce6d6269a59a4014673e41768 Mon Sep 17 00:00:00 2001 From: Dean Gardiner Date: Fri, 13 Dec 2013 14:02:07 +1300 Subject: [PATCH 5/6] Updated Caper to v0.3.1 --- libs/caper/__init__.py | 24 +++- libs/caper/constraint.py | 111 ++++++++++++++---- libs/caper/group.py | 220 ++++++++++++++++++++++++++++------- libs/caper/matcher.py | 2 +- libs/caper/objects.py | 43 +++++++ libs/caper/parsers/anime.py | 4 +- libs/caper/parsers/base.py | 14 ++- libs/caper/parsers/scene.py | 16 +-- libs/caper/parsers/usenet.py | 115 ++++++++++++++++++ libs/caper/result.py | 95 ++++++++++----- libs/caper/step.py | 54 +++++++-- 11 files changed, 574 insertions(+), 124 deletions(-) create mode 100644 libs/caper/parsers/usenet.py diff --git a/libs/caper/__init__.py b/libs/caper/__init__.py index 8b2e61a0..95fb6d73 100644 --- a/libs/caper/__init__.py +++ b/libs/caper/__init__.py @@ -17,9 +17,10 @@ from caper.matcher import FragmentMatcher from caper.objects import CaperFragment, CaperClosure from caper.parsers.anime import AnimeParser from caper.parsers.scene import SceneParser +from caper.parsers.usenet import UsenetParser -__version_info__ = ('0', '2', '9') +__version_info__ = ('0', '3', '1') __version_branch__ = 'master' __version__ = "%s%s" % ( @@ -28,8 +29,9 @@ __version__ = "%s%s" % ( ) -CL_START_CHARS = ['(', '['] -CL_END_CHARS = [')', ']'] +CL_START_CHARS = ['(', '[', '<', '>'] +CL_END_CHARS = [')', ']', '<', '>'] +CL_END_STRINGS = [' - '] STRIP_START_CHARS = ''.join(CL_START_CHARS) STRIP_END_CHARS = ''.join(CL_END_CHARS) @@ -47,8 +49,9 @@ class Caper(object): self.debug = debug self.parsers = { + 'anime': AnimeParser, 'scene': SceneParser, - 'anime': AnimeParser + 'usenet': UsenetParser } def _closure_split(self, name): @@ -62,7 +65,7 @@ class Caper(object): def end_closure(closures, buf): buf = buf.strip(STRIP_CHARS) - if len(buf) < 1: + if len(buf) < 2: return cur = CaperClosure(len(closures), buf) @@ -76,6 +79,7 @@ class Caper(object): state = CL_START buf = "" for x, ch in enumerate(name): + # Check for start characters if state == CL_START and ch in CL_START_CHARS: end_closure(closures, buf) @@ -85,8 +89,15 @@ class Caper(object): buf += ch if state == CL_END and ch in CL_END_CHARS: + # End character found, create the closure end_closure(closures, buf) + state = CL_START + buf = "" + elif state == CL_START and buf[-3:] in CL_END_STRINGS: + # End string found, create the closure + end_closure(closures, buf[:-3]) + state = CL_START buf = "" @@ -174,6 +185,9 @@ class Caper(object): for closure in closures: Logr.debug("closure [%s]", closure.value) + for fragment in closure.fragments: + Logr.debug("\tfragment [%s]", fragment.value) + if parser not in self.parsers: raise ValueError("Unknown parser") diff --git a/libs/caper/constraint.py b/libs/caper/constraint.py index 96f45c35..e092d33d 100644 --- a/libs/caper/constraint.py +++ b/libs/caper/constraint.py @@ -14,7 +14,7 @@ class CaptureConstraint(object): - def __init__(self, capture_group, comparisons=None, **kwargs): + def __init__(self, capture_group, constraint_type, comparisons=None, target=None, **kwargs): """Capture constraint object :type capture_group: CaptureGroup @@ -22,50 +22,113 @@ class CaptureConstraint(object): self.capture_group = capture_group - self.comparisons = comparisons if comparisons else [] + self.constraint_type = constraint_type + self.target = target - for key, value in kwargs.items(): - key = key.split('__') + self.comparisons = comparisons if comparisons else [] + self.kwargs = {} + + for orig_key, value in kwargs.items(): + key = orig_key.split('__') if len(key) != 2: + self.kwargs[orig_key] = value continue name, method = key - method = '_compare_' + method + method = 'constraint_match_' + method if not hasattr(self, method): + self.kwargs[orig_key] = value continue self.comparisons.append((name, getattr(self, method), value)) - def _compare_eq(self, fragment, name, expected): - if not hasattr(fragment, name): - return 1.0, False + def execute(self, parent_node, node, **kwargs): + func_name = 'constraint_%s' % self.constraint_type - return 1.0, getattr(fragment, name) == expected + if hasattr(self, func_name): + return getattr(self, func_name)(parent_node, node, **kwargs) - def _compare_re(self, fragment, name, arg): - if name == 'fragment': - group, minimum_weight = arg if type(arg) is tuple and len(arg) > 1 else (arg, 0) + raise ValueError('Unknown constraint type "%s"' % self.constraint_type) - weight, match, num_fragments = self.capture_group.parser.matcher.fragment_match(fragment, group) - return weight, weight > minimum_weight - elif type(arg).__name__ == 'SRE_Pattern': - return 1.0, arg.match(getattr(fragment, name)) is not None - elif hasattr(fragment, name): - match = self.capture_group.parser.matcher.value_match(getattr(fragment, name), arg, single=True) - return 1.0, match is not None - else: - raise ValueError("Unable to find attribute with name '%s'" % name) + # + # Node Matching + # - def execute(self, fragment): + def constraint_match(self, parent_node, node): results = [] total_weight = 0 for name, method, argument in self.comparisons: - weight, success = method(fragment, name, argument) + weight, success = method(node, name, argument) total_weight += weight results.append(success) - return total_weight / float(len(results)), all(results) if len(results) > 0 else False + return total_weight / (float(len(results)) or 1), all(results) if len(results) > 0 else False + + def constraint_match_eq(self, node, name, expected): + if not hasattr(node, name): + return 1.0, False + + return 1.0, getattr(node, name) == expected + + def constraint_match_re(self, node, name, arg): + # Node match + if name == 'node': + group, minimum_weight = arg if type(arg) is tuple and len(arg) > 1 else (arg, 0) + + weight, match, num_fragments = self.capture_group.parser.matcher.fragment_match(node, group) + return weight, weight > minimum_weight + + # Regex match + if type(arg).__name__ == 'SRE_Pattern': + return 1.0, arg.match(getattr(node, name)) is not None + + # Value match + if hasattr(node, name): + match = self.capture_group.parser.matcher.value_match(getattr(node, name), arg, single=True) + return 1.0, match is not None + + raise ValueError("Unknown constraint match type '%s'" % name) + + # + # Result + # + + def constraint_result(self, parent_node, fragment): + ctag = self.kwargs.get('tag') + if not ctag: + return 0, False + + ckey = self.kwargs.get('key') + + for tag, result in parent_node.captured(): + if tag != ctag: + continue + + if not ckey or ckey in result.keys(): + return 1.0, True + + return 0.0, False + + # + # Failure + # + + def constraint_failure(self, parent_node, fragment, match): + if not match or not match.success: + return 1.0, True + + return 0, False + + # + # Success + # + + def constraint_success(self, parent_node, fragment, match): + if match and match.success: + return 1.0, True + + return 0, False def __repr__(self): return "CaptureConstraint(comparisons=%s)" % repr(self.comparisons) diff --git a/libs/caper/group.py b/libs/caper/group.py index 71b97664..8f0399ef 100644 --- a/libs/caper/group.py +++ b/libs/caper/group.py @@ -14,7 +14,7 @@ from logr import Logr -from caper import CaperClosure +from caper import CaperClosure, CaperFragment from caper.helpers import clean_dict from caper.result import CaperFragmentNode, CaperClosureNode from caper.step import CaptureStep @@ -34,86 +34,214 @@ class CaptureGroup(object): #: @type: list of CaptureStep self.steps = [] - #: @type: list of CaptureConstraint - self.constraints = [] - def capture_fragment(self, tag, regex=None, func=None, single=True): + #: type: str + self.step_source = None + + #: @type: list of CaptureConstraint + self.pre_constraints = [] + + #: :type: list of CaptureConstraint + self.post_constraints = [] + + def capture_fragment(self, tag, regex=None, func=None, single=True, **kwargs): Logr.debug('capture_fragment("%s", "%s", %s, %s)', tag, regex, func, single) + if self.step_source != 'fragment': + if self.step_source is None: + self.step_source = 'fragment' + else: + raise ValueError("Unable to mix fragment and closure capturing in a group") + self.steps.append(CaptureStep( self, tag, 'fragment', regex=regex, func=func, - single=single + single=single, + **kwargs )) return self - def capture_closure(self, tag, regex=None, func=None, single=True): + def capture_closure(self, tag, regex=None, func=None, single=True, **kwargs): Logr.debug('capture_closure("%s", "%s", %s, %s)', tag, regex, func, single) + if self.step_source != 'closure': + if self.step_source is None: + self.step_source = 'closure' + else: + raise ValueError("Unable to mix fragment and closure capturing in a group") + self.steps.append(CaptureStep( self, tag, 'closure', regex=regex, func=func, - single=single + single=single, + **kwargs )) return self - def until(self, **kwargs): - self.constraints.append(CaptureConstraint(self, **kwargs)) + def until_closure(self, **kwargs): + self.pre_constraints.append(CaptureConstraint(self, 'match', target='closure', **kwargs)) + + return self + + def until_fragment(self, **kwargs): + self.pre_constraints.append(CaptureConstraint(self, 'match', target='fragment', **kwargs)) + + return self + + def until_result(self, **kwargs): + self.pre_constraints.append(CaptureConstraint(self, 'result', **kwargs)) + + return self + + def until_failure(self, **kwargs): + self.post_constraints.append(CaptureConstraint(self, 'failure', **kwargs)) + + return self + + def until_success(self, **kwargs): + self.post_constraints.append(CaptureConstraint(self, 'success', **kwargs)) return self def parse_subject(self, parent_head, subject): + Logr.debug("parse_subject (%s) subject: %s", self.step_source, repr(subject)) + + if type(subject) is CaperClosure: + return self.parse_closure(parent_head, subject) + + if type(subject) is CaperFragment: + return self.parse_fragment(parent_head, subject) + + raise ValueError('Unknown subject (%s)', subject) + + def parse_fragment(self, parent_head, subject): parent_node = parent_head[0] if type(parent_head) is list else parent_head - # TODO just jumping into closures for now, will be fixed later - if type(subject) is CaperClosure: - return [CaperClosureNode(subject, parent_head)] + nodes, match = self.match(parent_head, parent_node, subject) - nodes = [] - - # Check constraints - for constraint in self.constraints: - weight, success = constraint.execute(subject) - if success: - Logr.debug('capturing broke on "%s" at %s', subject.value, constraint) - parent_node.finished_groups.append(self) - nodes.append(parent_head) - - if weight == 1.0: - return nodes - else: - Logr.debug('Branching result') - - # Try match subject against the steps available - tag, success, weight, match, num_fragments = (None, None, None, None, None) - for step in self.steps: - tag = step.tag - success, weight, match, num_fragments = step.execute(subject) - if success: - match = clean_dict(match) if type(match) is dict else match - Logr.debug('Found match with weight %s, match: %s, num_fragments: %s' % (weight, match, num_fragments)) - break + # Capturing broke on constraint, return now + if not match: + return nodes Logr.debug('created fragment node with subject.value: "%s"' % subject.value) - result = [CaperFragmentNode(parent_node.closure, subject.take_right(num_fragments), parent_head, tag, weight, match)] + result = [CaperFragmentNode( + parent_node.closure, + subject.take_right(match.num_fragments), + parent_head, + match + )] - if match and weight < 1.0: - if num_fragments == 1: - result.append(CaperFragmentNode(parent_node.closure, [subject], parent_head, None, None, None)) + # Branch if the match was indefinite (weight below 1.0) + if match.result and match.weight < 1.0: + if match.num_fragments == 1: + result.append(CaperFragmentNode(parent_node.closure, [subject], parent_head)) else: - nodes.append(CaperFragmentNode(parent_node.closure, [subject], parent_head, None, None, None)) + nodes.append(CaperFragmentNode(parent_node.closure, [subject], parent_head)) nodes.append(result[0] if len(result) == 1 else result) return nodes + def parse_closure(self, parent_head, subject): + parent_node = parent_head[0] if type(parent_head) is list else parent_head + + nodes, match = self.match(parent_head, parent_node, subject) + + # Capturing broke on constraint, return now + if not match: + return nodes + + Logr.debug('created closure node with subject.value: "%s"' % subject.value) + + result = [CaperClosureNode( + subject, + parent_head, + match + )] + + # Branch if the match was indefinite (weight below 1.0) + if match.result and match.weight < 1.0: + if match.num_fragments == 1: + result.append(CaperClosureNode(subject, parent_head)) + else: + nodes.append(CaperClosureNode(subject, parent_head)) + + nodes.append(result[0] if len(result) == 1 else result) + + return nodes + + def match(self, parent_head, parent_node, subject): + nodes = [] + + # Check pre constaints + broke, definite = self.check_constraints(self.pre_constraints, parent_head, subject) + + if broke: + nodes.append(parent_head) + + if definite: + return nodes, None + + # Try match subject against the steps available + match = None + + for step in self.steps: + if step.source == 'closure' and type(subject) is not CaperClosure: + pass + elif step.source == 'fragment' and type(subject) is CaperClosure: + Logr.debug('Closure encountered on fragment step, jumping into fragments') + return [CaperClosureNode(subject, parent_head, None)], None + + match = step.execute(subject) + + if match.success: + if type(match.result) is dict: + match.result = clean_dict(match.result) + + Logr.debug('Found match with weight %s, match: %s, num_fragments: %s' % ( + match.weight, match.result, match.num_fragments + )) + + step.matched = True + + break + + if all([step.single and step.matched for step in self.steps]): + Logr.debug('All steps completed, group finished') + parent_node.finished_groups.append(self) + return nodes, match + + # Check post constraints + broke, definite = self.check_constraints(self.post_constraints, parent_head, subject, match=match) + if broke: + return nodes, None + + return nodes, match + + def check_constraints(self, constraints, parent_head, subject, **kwargs): + parent_node = parent_head[0] if type(parent_head) is list else parent_head + + # Check constraints + for constraint in [c for c in constraints if c.target == subject.__key__ or not c.target]: + Logr.debug("Testing constraint %s against subject %s", repr(constraint), repr(subject)) + + weight, success = constraint.execute(parent_node, subject, **kwargs) + + if success: + Logr.debug('capturing broke on "%s" at %s', subject.value, constraint) + parent_node.finished_groups.append(self) + + return True, weight == 1.0 + + return False, None + def execute(self): heads_finished = None @@ -126,20 +254,26 @@ class CaptureGroup(object): for head in heads: node = head[0] if type(head) is list else head - Logr.debug("head node: %s" % node) - if self in node.finished_groups: Logr.debug("head finished for group") self.result.heads.append(head) heads_finished.append(True) continue + Logr.debug('') + + Logr.debug(node) + next_subject = node.next() + Logr.debug('----------[%s] (%s)----------' % (next_subject, repr(next_subject.value) if next_subject else None)) + if next_subject: for node_result in self.parse_subject(head, next_subject): self.result.heads.append(node_result) + Logr.debug('Heads: %s', self.result.heads) + heads_finished.append(self in node.finished_groups or next_subject is None) if len(self.result.heads) == 0: diff --git a/libs/caper/matcher.py b/libs/caper/matcher.py index c154cd70..3acf2e68 100644 --- a/libs/caper/matcher.py +++ b/libs/caper/matcher.py @@ -71,7 +71,7 @@ class FragmentMatcher(object): if group_name and group_name == name: return group_name, weight_groups - return None + return None, None def value_match(self, value, group_name=None, single=True): result = None diff --git a/libs/caper/objects.py b/libs/caper/objects.py index 1f82c33e..b7d9084d 100644 --- a/libs/caper/objects.py +++ b/libs/caper/objects.py @@ -16,6 +16,8 @@ from caper.helpers import xrange_six class CaperClosure(object): + __key__ = 'closure' + def __init__(self, index, value): #: :type: int self.index = index @@ -31,8 +33,16 @@ class CaperClosure(object): #: :type: list of CaperFragment self.fragments = [] + def __str__(self): + return "" % repr(self.result) + + def __repr__(self): + return self.__str__() diff --git a/libs/caper/parsers/anime.py b/libs/caper/parsers/anime.py index 88313a2c..86c70917 100644 --- a/libs/caper/parsers/anime.py +++ b/libs/caper/parsers/anime.py @@ -75,8 +75,8 @@ class AnimeParser(Parser): .execute(once=True) self.capture_fragment('show_name', single=False)\ - .until(value__re='identifier')\ - .until(value__re='video')\ + .until_fragment(value__re='identifier')\ + .until_fragment(value__re='video')\ .execute() self.capture_fragment('identifier', regex='identifier') \ diff --git a/libs/caper/parsers/base.py b/libs/caper/parsers/base.py index 6bae537c..16bbc19f 100644 --- a/libs/caper/parsers/base.py +++ b/libs/caper/parsers/base.py @@ -14,7 +14,7 @@ from caper import FragmentMatcher from caper.group import CaptureGroup -from caper.result import CaperResult, CaperClosureNode +from caper.result import CaperResult, CaperClosureNode, CaperRootNode from logr import Logr @@ -52,7 +52,7 @@ class Parser(object): self.reset() self.closures = closures - self.result.heads = [CaperClosureNode(closures[0])] + self.result.heads = [CaperRootNode(closures[0])] def run(self, closures): """ @@ -65,18 +65,20 @@ class Parser(object): # Capture Methods # - def capture_fragment(self, tag, regex=None, func=None, single=True): + def capture_fragment(self, tag, regex=None, func=None, single=True, **kwargs): return CaptureGroup(self, self.result).capture_fragment( tag, regex=regex, func=func, - single=single + single=single, + **kwargs ) - def capture_closure(self, tag, regex=None, func=None, single=True): + def capture_closure(self, tag, regex=None, func=None, single=True, **kwargs): return CaptureGroup(self, self.result).capture_closure( tag, regex=regex, func=func, - single=single + single=single, + **kwargs ) diff --git a/libs/caper/parsers/scene.py b/libs/caper/parsers/scene.py index 0dfe378a..cd0a8fdf 100644 --- a/libs/caper/parsers/scene.py +++ b/libs/caper/parsers/scene.py @@ -185,11 +185,11 @@ class SceneParser(Parser): self.setup(closures) self.capture_fragment('show_name', single=False)\ - .until(fragment__re='identifier')\ - .until(fragment__re='video')\ - .until(fragment__re='dvd')\ - .until(fragment__re='audio')\ - .until(fragment__re='scene')\ + .until_fragment(node__re='identifier')\ + .until_fragment(node__re='video')\ + .until_fragment(node__re='dvd')\ + .until_fragment(node__re='audio')\ + .until_fragment(node__re='scene')\ .execute() self.capture_fragment('identifier', regex='identifier', single=False)\ @@ -197,7 +197,7 @@ class SceneParser(Parser): .capture_fragment('dvd', regex='dvd', single=False)\ .capture_fragment('audio', regex='audio', single=False)\ .capture_fragment('scene', regex='scene', single=False)\ - .until(left_sep__eq='-', right__eq=None)\ + .until_fragment(left_sep__eq='-', right__eq=None)\ .execute() self.capture_fragment('group', func=self.capture_group)\ @@ -222,7 +222,9 @@ class SceneParser(Parser): Logr.debug(head[0].closure.value) for node in head: - Logr.debug('\t' + str(node).ljust(55) + '\t' + str(node.weight) + '\t' + str(node.match)) + Logr.debug('\t' + str(node).ljust(55) + '\t' + ( + str(node.match.weight) + '\t' + str(node.match.result) + ) if node.match else '') if len(head) > 0 and head[0].parent: self.print_tree([head[0].parent]) diff --git a/libs/caper/parsers/usenet.py b/libs/caper/parsers/usenet.py new file mode 100644 index 00000000..f622d43b --- /dev/null +++ b/libs/caper/parsers/usenet.py @@ -0,0 +1,115 @@ +# Copyright 2013 Dean Gardiner +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from logr import Logr +from caper import FragmentMatcher +from caper.parsers.base import Parser + + +PATTERN_GROUPS = [ + ('usenet', [ + r'\[(?P#[\w\.@]+)\]', + r'^\[(?P\w+)\]$', + r'\[(?PFULL)\]', + r'\[\s?(?PTOWN)\s?\]', + r'(.*?\s)?[_\W]*(?Pwww\..*?\.[a-z0-9]+)[_\W]*(.*?\s)?', + r'(.*?\s)?[_\W]*(?P(www\.)?[-\w]+\.(com|org|info))[_\W]*(.*?\s)?' + ]), + + ('part', [ + r'.?(?P\d+)/(?P\d+).?' + ]), + + ('detail', [ + r'[\s-]*\w*?[\s-]*\"(?P.*?)\"[\s-]*\w*?[\s-]*(?P[\d,\.]*\s?MB)?[\s-]*(?PyEnc)?', + r'(?P[\d,\.]*\s?MB)[\s-]*(?PyEnc)', + r'(?P[\d,\.]*\s?MB)|(?PyEnc)' + ]) +] + + +class UsenetParser(Parser): + matcher = None + + def __init__(self, debug=False): + if not UsenetParser.matcher: + UsenetParser.matcher = FragmentMatcher(PATTERN_GROUPS) + Logr.info("Fragment matcher for %s created", self.__class__.__name__) + + super(UsenetParser, self).__init__(UsenetParser.matcher, debug) + + def run(self, closures): + """ + :type closures: list of CaperClosure + """ + + self.setup(closures) + + # Capture usenet or part info until we get a part or matching fails + self.capture_closure('usenet', regex='usenet', single=False)\ + .capture_closure('part', regex='part', single=True) \ + .until_result(tag='part') \ + .until_failure()\ + .execute() + + is_town_release, has_part = self.get_state() + + if not is_town_release: + self.capture_release_name() + + # If we already have the part (TOWN releases), ignore matching part again + if not is_town_release and not has_part: + self.capture_fragment('part', regex='part', single=True)\ + .until_closure(node__re='usenet')\ + .until_success()\ + .execute() + + # Capture any leftover details + self.capture_closure('usenet', regex='usenet', single=False)\ + .capture_closure('detail', regex='detail', single=False)\ + .execute() + + self.result.build() + return self.result + + def capture_release_name(self): + self.capture_closure('detail', regex='detail', single=False)\ + .until_failure()\ + .execute() + + self.capture_fragment('release_name', single=False, include_separators=True) \ + .until_closure(node__re='usenet') \ + .until_closure(node__re='detail') \ + .until_closure(node__re='part') \ + .until_fragment(value__eq='-')\ + .execute() + + # Capture any detail after the release name + self.capture_closure('detail', regex='detail', single=False)\ + .until_failure()\ + .execute() + + def get_state(self): + # TODO multiple-chains? + is_town_release = False + has_part = False + + for tag, result in self.result.heads[0].captured(): + if tag == 'usenet' and result.get('group') == 'TOWN': + is_town_release = True + + if tag == 'part': + has_part = True + + return is_town_release, has_part diff --git a/libs/caper/result.py b/libs/caper/result.py index 24037cdf..c9e34237 100644 --- a/libs/caper/result.py +++ b/libs/caper/result.py @@ -20,7 +20,7 @@ GROUP_MATCHES = ['identifier'] class CaperNode(object): - def __init__(self, closure, parent=None, tag=None, weight=None, match=None): + def __init__(self, closure, parent=None, match=None): """ :type parent: CaperNode :type weight: float @@ -28,41 +28,77 @@ class CaperNode(object): #: :type: caper.objects.CaperClosure self.closure = closure + #: :type: CaperNode self.parent = parent - #: :type: str - self.tag = tag - #: :type: float - self.weight = weight - #: :type: dict + + #: :type: CaptureMatch self.match = match + #: :type: list of CaptureGroup self.finished_groups = [] def next(self): raise NotImplementedError() + def captured(self): + cur = self -class CaperClosureNode(CaperNode): - def __init__(self, closure, parent=None, tag=None, weight=None, match=None): + if cur.match: + yield cur.match.tag, cur.match.result + + while cur.parent: + cur = cur.parent + + if cur.match: + yield cur.match.tag, cur.match.result + + +class CaperRootNode(CaperNode): + def __init__(self, closure): """ :type closure: caper.objects.CaperClosure or list of caper.objects.CaperClosure """ - super(CaperClosureNode, self).__init__(closure, parent, tag, weight, match) + super(CaperRootNode, self).__init__(closure) def next(self): - if self.closure and len(self.closure.fragments) > 0: + return self.closure + + +class CaperClosureNode(CaperNode): + def __init__(self, closure, parent=None, match=None): + """ + :type closure: caper.objects.CaperClosure or list of caper.objects.CaperClosure + """ + super(CaperClosureNode, self).__init__(closure, parent, match) + + def next(self): + if not self.closure: + return None + + if self.match: + # Jump to next closure if we have a match + return self.closure.right + elif len(self.closure.fragments) > 0: + # Otherwise parse the fragments return self.closure.fragments[0] + return None + def __str__(self): + return "" % repr(self.match) + + def __repr__(self): + return self.__str__() + class CaperFragmentNode(CaperNode): - def __init__(self, closure, fragments, parent=None, tag=None, weight=None, match=None): + def __init__(self, closure, fragments, parent=None, match=None): """ :type closure: caper.objects.CaperClosure :type fragments: list of caper.objects.CaperFragment """ - super(CaperFragmentNode, self).__init__(closure, parent, tag, weight, match) + super(CaperFragmentNode, self).__init__(closure, parent, match) #: :type: caper.objects.CaperFragment or list of caper.objects.CaperFragment self.fragments = fragments @@ -76,6 +112,12 @@ class CaperFragmentNode(CaperNode): return None + def __str__(self): + return "" % repr(self.match) + + def __repr__(self): + return self.__str__() + class CaperResult(object): def __init__(self): @@ -122,15 +164,8 @@ class CaperResult(object): result.append(node_chain) continue - # Skip over closure nodes - if type(node) is CaperClosureNode: - result.extend(self.combine_chain(node.parent, node_chain)) - - # Parse fragment matches - if type(node) is CaperFragmentNode: - node_chain.update(node) - - result.extend(self.combine_chain(node.parent, node_chain)) + node_chain.update(node) + result.extend(self.combine_chain(node.parent, node_chain)) return result @@ -145,17 +180,23 @@ class CaperResultChain(object): self.weights = [] def update(self, subject): - if subject.weight is None: + """ + :type subject: CaperFragmentNode + """ + if not subject.match or not subject.match.success: return - self.num_matched += len(subject.fragments) if subject.fragments is not None else 0 - self.weights.append(subject.weight) + # TODO this should support closure nodes + if type(subject) is CaperFragmentNode: + self.num_matched += len(subject.fragments) if subject.fragments is not None else 0 + + self.weights.append(subject.match.weight) if subject.match: - if subject.tag not in self.info: - self.info[subject.tag] = [] + if subject.match.tag not in self.info: + self.info[subject.match.tag] = [] - self.info[subject.tag].insert(0, subject.match) + self.info[subject.match.tag].insert(0, subject.match.result) def finish(self): self.weight = sum(self.weights) / len(self.weights) diff --git a/libs/caper/step.py b/libs/caper/step.py index a82a9301..817514b6 100644 --- a/libs/caper/step.py +++ b/libs/caper/step.py @@ -12,13 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. +from caper.objects import CaptureMatch from logr import Logr class CaptureStep(object): REPR_KEYS = ['regex', 'func', 'single'] - def __init__(self, capture_group, tag, source, regex=None, func=None, single=None): + def __init__(self, capture_group, tag, source, regex=None, func=None, single=None, **kwargs): #: @type: CaptureGroup self.capture_group = capture_group @@ -33,22 +34,57 @@ class CaptureStep(object): #: @type: bool self.single = single + self.kwargs = kwargs + + self.matched = False + def execute(self, fragment): + """Execute step on fragment + + :type fragment: CaperFragment + :rtype : CaptureMatch + """ + + match = CaptureMatch(self.tag, self) + if self.regex: - weight, match, num_fragments = self.capture_group.parser.matcher.fragment_match(fragment, self.regex) + weight, result, num_fragments = self.capture_group.parser.matcher.fragment_match(fragment, self.regex) Logr.debug('(execute) [regex] tag: "%s"', self.tag) - if match: - return True, weight, match, num_fragments + + if not result: + return match + + # Populate CaptureMatch + match.success = True + match.weight = weight + match.result = result + match.num_fragments = num_fragments elif self.func: - match = self.func(fragment) + result = self.func(fragment) Logr.debug('(execute) [func] %s += "%s"', self.tag, match) - if match: - return True, 1.0, match, 1 + + if not result: + return match + + # Populate CaptureMatch + match.success = True + match.weight = 1.0 + match.result = result else: Logr.debug('(execute) [raw] %s += "%s"', self.tag, fragment.value) - return True, 1.0, fragment.value, 1 - return False, None, None, 1 + include_separators = self.kwargs.get('include_separators', False) + + # Populate CaptureMatch + match.success = True + match.weight = 1.0 + + if include_separators: + match.result = (fragment.left_sep, fragment.value, fragment.right_sep) + else: + match.result = fragment.value + + return match def __repr__(self): attribute_values = [key + '=' + repr(getattr(self, key)) From 242d69a98165ad4e79896dfeaf3d11dd339f129c Mon Sep 17 00:00:00 2001 From: Dean Gardiner Date: Fri, 13 Dec 2013 16:31:26 +1300 Subject: [PATCH 6/6] The nzbindex provider now uses the caper usenet parser to get release names from usenet subjects. --- couchpotato/core/media/_base/matcher/base.py | 38 ++++++++++++++++--- couchpotato/core/media/_base/matcher/main.py | 3 ++ .../core/providers/nzb/nzbindex/main.py | 33 +++++++++++++++- 3 files changed, 66 insertions(+), 8 deletions(-) diff --git a/couchpotato/core/media/_base/matcher/base.py b/couchpotato/core/media/_base/matcher/base.py index c4b59b25..86511263 100644 --- a/couchpotato/core/media/_base/matcher/base.py +++ b/couchpotato/core/media/_base/matcher/base.py @@ -17,16 +17,42 @@ class MatcherBase(Plugin): raise NotImplementedError() def flattenInfo(self, info): - flat_info = {} + # Flatten dictionary of matches (chain info) + if isinstance(info, dict): + return dict([(key, self.flattenInfo(value)) for key, value in info.items()]) + + # Flatten matches + result = None for match in info: - for key, value in match.items(): - if key not in flat_info: - flat_info[key] = [] + if isinstance(match, dict): + if result is None: + result = {} - flat_info[key].append(value) + for key, value in match.items(): + if key not in result: + result[key] = [] - return flat_info + result[key].append(value) + else: + if result is None: + result = [] + + result.append(match) + + return result + + def constructFromRaw(self, match): + if not match: + return None + + parts = [ + ''.join([ + y for y in x[1:] if y + ]) for x in match + ] + + return ''.join(parts)[:-1].strip() def simplifyValue(self, value): if not value: diff --git a/couchpotato/core/media/_base/matcher/main.py b/couchpotato/core/media/_base/matcher/main.py index 9cc8c1a0..12e3d6a1 100644 --- a/couchpotato/core/media/_base/matcher/main.py +++ b/couchpotato/core/media/_base/matcher/main.py @@ -16,6 +16,9 @@ class Matcher(MatcherBase): addEvent('matcher.parse', self.parse) addEvent('matcher.match', self.match) + addEvent('matcher.flatten_info', self.flattenInfo) + addEvent('matcher.construct_from_raw', self.constructFromRaw) + addEvent('matcher.correct_title', self.correctTitle) addEvent('matcher.correct_quality', self.correctQuality) diff --git a/couchpotato/core/providers/nzb/nzbindex/main.py b/couchpotato/core/providers/nzb/nzbindex/main.py index abc59a5c..73435420 100644 --- a/couchpotato/core/providers/nzb/nzbindex/main.py +++ b/couchpotato/core/providers/nzb/nzbindex/main.py @@ -38,6 +38,35 @@ class Base(NZBProvider, RSS): enclosure = self.getElement(nzb, 'enclosure').attrib nzbindex_id = int(self.getTextElement(nzb, "link").split('/')[4]) + title = self.getTextElement(nzb, "title") + + match = fireEvent('matcher.parse', title, parser='usenet', single = True) + if not match.chains: + log.info('Unable to parse release with title "%s"', title) + continue + + # TODO should we consider other lower-weight chains here? + info = fireEvent('matcher.flatten_info', match.chains[0].info, single = True) + + release_name = fireEvent('matcher.construct_from_raw', info.get('release_name'), single = True) + + file_name = info.get('detail', {}).get('file_name') + file_name = file_name[0] if file_name else None + + title = release_name or file_name + + # Strip extension from parsed title (if one exists) + ext_pos = title.rfind('.') + + # Assume extension if smaller than 4 characters + # TODO this should probably be done a better way + if len(title[ext_pos + 1:]) <= 4: + title = title[:ext_pos] + + if not title: + log.info('Unable to find release name from match') + continue + try: description = self.getTextElement(nzb, "description") except: @@ -52,7 +81,7 @@ class Base(NZBProvider, RSS): results.append({ 'id': nzbindex_id, - 'name': self.getTextElement(nzb, "title"), + 'name': title, 'age': self.calculateAge(int(time.mktime(parse(self.getTextElement(nzb, "pubDate")).timetuple()))), 'size': tryInt(enclosure['length']) / 1024 / 1024, 'url': enclosure['url'], @@ -74,7 +103,7 @@ class Base(NZBProvider, RSS): class Movie(MovieProvider, Base): - def buildUrl(self, media): + def buildUrl(self, media, quality): title = fireEvent('searcher.get_search_title', media['library'], single = True) year = media['library']['year']