diff --git a/couchpotato/core/media/__init__.py b/couchpotato/core/media/__init__.py index 1ba83863..3894fbf8 100644 --- a/couchpotato/core/media/__init__.py +++ b/couchpotato/core/media/__init__.py @@ -1,5 +1,6 @@ from couchpotato import get_session from couchpotato.core.event import addEvent, fireEventAsync, fireEvent +from couchpotato.core.helpers.variable import mergeDicts from couchpotato.core.plugins.base import Plugin from couchpotato.core.settings.model import Media @@ -17,6 +18,13 @@ class MediaBase(Plugin): 'category': {}, } + search_dict = mergeDicts(default_dict, { + 'library': { + 'related_libraries': {}, + 'root_library': {} + }, + }) + def initType(self): addEvent('media.types', self.getType) @@ -28,7 +36,7 @@ class MediaBase(Plugin): def onComplete(): db = get_session() media = db.query(Media).filter_by(id = id).first() - fireEventAsync('%s.searcher.single' % media.type, media.to_dict(self.default_dict), on_complete = self.createNotifyFront(id)) + fireEventAsync('%s.searcher.single' % media.type, media.to_dict(self.search_dict), on_complete = self.createNotifyFront(id)) db.expire_all() return onComplete diff --git a/couchpotato/core/media/show/library/episode/main.py b/couchpotato/core/media/show/library/episode/main.py index 07344a02..09cacd9d 100644 --- a/couchpotato/core/media/show/library/episode/main.py +++ b/couchpotato/core/media/show/library/episode/main.py @@ -17,10 +17,39 @@ class EpisodeLibraryPlugin(LibraryBase): default_dict = {'titles': {}, 'files':{}} def __init__(self): + addEvent('library.identifier', self.identifier) addEvent('library.add.episode', self.add) addEvent('library.update.episode', self.update) addEvent('library.update.episode_release_date', self.updateReleaseDate) + def identifier(self, library): + if library.get('type') != 'episode': + return + + identifier = { + 'season': None, + 'episode': None + } + + scene_map = library['info'].get('map_episode', {}).get('scene') + + if scene_map: + # Use scene mappings if they are available + identifier['season'] = scene_map.get('season') + identifier['episode'] = scene_map.get('episode') + else: + # Fallback to normal season/episode numbers + identifier['season'] = library.get('season_number') + identifier['episode'] = library.get('episode_number') + + + # Cast identifiers to integers + # TODO this will need changing to support identifiers with trailing 'a', 'b' characters + identifier['season'] = tryInt(identifier['season'], None) + identifier['episode'] = tryInt(identifier['episode'], None) + + return identifier + def add(self, attrs = {}, update_after = True): type = attrs.get('type', 'episode') primary_provider = attrs.get('primary_provider', 'thetvdb') diff --git a/couchpotato/core/media/show/library/season/main.py b/couchpotato/core/media/show/library/season/main.py index 75f19d5a..48d201ed 100644 --- a/couchpotato/core/media/show/library/season/main.py +++ b/couchpotato/core/media/show/library/season/main.py @@ -17,10 +17,22 @@ class SeasonLibraryPlugin(LibraryBase): default_dict = {'titles': {}, 'files':{}} def __init__(self): + addEvent('library.identifier', self.identifier) addEvent('library.add.season', self.add) addEvent('library.update.season', self.update) addEvent('library.update.season_release_date', self.updateReleaseDate) + def identifier(self, library): + if library.get('type') != 'season': + return + + season_num = tryInt(library['season_number'], None) + + return { + 'season': season_num, + 'episode': None + } + def add(self, attrs = {}, update_after = True): type = attrs.get('type', 'season') primary_provider = attrs.get('primary_provider', 'thetvdb') diff --git a/couchpotato/core/media/show/searcher/main.py b/couchpotato/core/media/show/searcher/main.py index 4ed791f7..c4aca7b7 100644 --- a/couchpotato/core/media/show/searcher/main.py +++ b/couchpotato/core/media/show/searcher/main.py @@ -1,10 +1,10 @@ -from couchpotato import get_session, Env +from couchpotato import Env, get_session from couchpotato.core.event import addEvent, fireEvent -from couchpotato.core.helpers.variable import getTitle, tryInt +from couchpotato.core.helpers.variable import getTitle, tryInt, toIterable from couchpotato.core.logger import CPLog from couchpotato.core.media._base.searcher.main import SearchSetupError from couchpotato.core.plugins.base import Plugin -from couchpotato.core.settings.model import Media, Library +from couchpotato.core.settings.model import Media from qcond import QueryCondenser from qcond.helpers import simplify @@ -13,6 +13,8 @@ log = CPLog(__name__) class ShowSearcher(Plugin): + type = ['show', 'season', 'episode'] + in_progress = False # TODO come back to this later, think this could be handled better @@ -29,16 +31,17 @@ class ShowSearcher(Plugin): self.query_condenser = QueryCondenser() - addEvent('show.searcher.single', self.single) + for type in toIterable(self.type): + addEvent('%s.searcher.single' % type, self.single) + addEvent('searcher.get_search_title', self.getSearchTitle) addEvent('searcher.correct_match', self.correctMatch) addEvent('searcher.correct_release', self.correctRelease) - addEvent('searcher.get_media_identifier', self.getMediaIdentifier) - addEvent('searcher.get_media_root', self.getMediaRoot) - def single(self, media, search_protocols = None, manual = False): + show, season, episode = self.getLibraries(media['library']) + if media['type'] == 'show': # TODO handle show searches (scan all seasons) return @@ -69,8 +72,7 @@ class ShowSearcher(Plugin): #fireEvent('episode.delete', episode['id'], single = True) return - show, season, episode = self.getMedia(media) - if show is None or season is None: + if not show or not season: log.error('Unable to find show or season library in database, missing required data for searching') return @@ -93,7 +95,7 @@ class ShowSearcher(Plugin): # Don't search for quality lower then already available. if has_better_quality is 0: - log.info('Search for %s S%02d%s in %s', (getTitle(show), season.season_number, "E%02d" % episode.episode_number if episode else "", quality_type['quality']['label'])) + log.info('Search for %s S%02d%s in %s', (getTitle(show), season['season_number'], "E%02d" % episode['episode_number'] if episode else "", quality_type['quality']['label'])) quality = fireEvent('quality.single', identifier = quality_type['quality']['identifier'], single = True) results = fireEvent('searcher.search', search_protocols, media, quality, single = True) @@ -135,15 +137,16 @@ class ShowSearcher(Plugin): if media['type'] not in ['show', 'season', 'episode']: return - show, season, episode = self.getMedia(media) - if show is None: + show, season, episode = self.getLibraries(media['library']) + + if not show: return None titles = [] # Add season map_names if they exist - if season is not None and 'map_names' in show.info: - season_names = show.info['map_names'].get(str(season.season_number), {}) + if season is not None and 'map_names' in show['info']: + season_names = show['info']['map_names'].get(str(season['season_number']), {}) # Add titles from all locations # TODO only add name maps from a specific location @@ -151,7 +154,7 @@ class ShowSearcher(Plugin): titles += [name for name in names if name not in titles] # Add show titles - titles += [title.title for title in show.titles if title.title not in titles] + titles += [title['title'] for title in show['titles'] if title['title'] not in titles] # Use QueryCondenser to build a list of optimal search titles condensed_titles = self.query_condenser.distinct(titles) @@ -170,9 +173,9 @@ class ShowSearcher(Plugin): return None # Add the identifier to search title - # TODO supporting other identifier formats - identifier = fireEvent('searcher.get_media_identifier', media['library'], single = True) + identifier = fireEvent('library.identifier', media['library'], single = True) + # TODO this needs to support other identifier formats if identifier['season']: title += ' S%02d' % identifier['season'] @@ -195,11 +198,7 @@ class ShowSearcher(Plugin): if not fireEvent('searcher.correct_words', release['name'], media, single = True): return False - show, season, episode = self.getMedia(media) - if show is None or season is None: - log.error('Unable to find show or season library in database, missing required data for searching') - return - + # TODO Matching is quite costly, maybe we should be caching release matches somehow? (also look at caper optimizations) match = fireEvent('matcher.best', release, media, quality, single = True) if match: return match.weight @@ -224,68 +223,24 @@ class ShowSearcher(Plugin): return True - # TODO move this somewhere else - def getMediaIdentifier(self, media_library): - if media_library['type'] not in ['show', 'season', 'episode']: - return None + def getLibraries(self, library): + if 'related_libraries' not in library: + log.warning("'related_libraries' missing from media library, unable to continue searching") + return None, None, None - identifier = { - 'season': None, - 'episode': None - } + libraries = library['related_libraries'] - if media_library['type'] == 'episode': - map_episode = media_library['info'].get('map_episode') + # Get libraries and return lists only if there is multiple items + show = libraries.get('show', []) + if len(show) <= 1: + show = show[0] if len(show) else None - if map_episode and 'scene' in map_episode: - identifier['season'] = map_episode['scene'].get('season') - identifier['episode'] = map_episode['scene'].get('episode') - else: - # TODO xem mapping? - identifier['season'] = media_library.get('season_number') - identifier['episode'] = media_library.get('episode_number') + season = libraries.get('season', []) + if len(season) <= 1: + season = season[0] if len(season) else None - if media_library['type'] == 'season': - identifier['season'] = media_library.get('season_number') - - # Try cast identifier values to integers - identifier['season'] = tryInt(identifier['season'], None) - identifier['episode'] = tryInt(identifier['episode'], None) - - return identifier - - # TODO move this somewhere else - def getMediaRoot(self, media): - if media['type'] not in ['show', 'season', 'episode']: - return None - - show, season, episode = self.getMedia(media) - if show is None or season is None: - log.error('Unable to find show or season library in database, missing required data for searching') - return - - return show.to_dict() - - # TODO move this somewhere else - def getMedia(self, media): - db = get_session() - - media_library = db.query(Library).filter_by(id = media['library_id']).first() - - show = None - season = None - episode = None - - if media['type'] == 'episode': - show = media_library.parent.parent - season = media_library.parent - episode = media_library - - if media['type'] == 'season': - show = media_library.parent - season = media_library - - if media['type'] == 'show': - show = media_library + episode = libraries.get('episode', []) + if len(episode) <= 1: + episode = episode[0] if len(episode) else None return show, season, episode diff --git a/couchpotato/core/plugins/matcher/main.py b/couchpotato/core/plugins/matcher/main.py index c20db2b1..777f91a0 100644 --- a/couchpotato/core/plugins/matcher/main.py +++ b/couchpotato/core/plugins/matcher/main.py @@ -33,7 +33,7 @@ class Matcher(Plugin): if fireEvent('searcher.correct_match', chain, release, media, quality, single = True): return chain - return None + return False def chainMatch(self, chain, group, tags): found_tags = [] @@ -50,7 +50,7 @@ class Matcher(Plugin): return set([key for key, value in tags.items() if None not in value]) == set(found_tags) def correctIdentifier(self, chain, media): - required_id = fireEvent('searcher.get_media_identifier', media['library'], single = True) + required_id = fireEvent('library.identifier', media['library'], single = True) if 'identifier' not in chain.info: return False @@ -73,7 +73,7 @@ class Matcher(Plugin): return True def correctTitle(self, chain, media): - root_library = fireEvent('searcher.get_media_root', media['library'], single = True) + root_library = media['library']['root_library'] if 'show_name' not in chain.info or not len(chain.info['show_name']): log.info('Wrong: missing show name in parsed result') diff --git a/couchpotato/core/providers/base.py b/couchpotato/core/providers/base.py index 75febaa8..82e03cec 100644 --- a/couchpotato/core/providers/base.py +++ b/couchpotato/core/providers/base.py @@ -105,7 +105,6 @@ class YarrProvider(Provider): type = 'movie' cat_ids = {} - cat_ids_structure = None cat_backup_id = None sizeGb = ['gb', 'gib'] @@ -250,33 +249,9 @@ class YarrProvider(Provider): return 0 - def _discoverCatIdStructure(self): - # Discover cat_ids structure (single or groups) - for group_name, group_cat_ids in self.cat_ids: - if len(group_cat_ids) > 0: - if type(group_cat_ids[0]) is tuple: - self.cat_ids_structure = 'group' - if type(group_cat_ids[0]) is str: - self.cat_ids_structure = 'single' + def getCatId(self, identifier): - def getCatId(self, identifier, group = None): - - cat_ids = self.cat_ids - - if not self.cat_ids_structure: - self._discoverCatIdStructure() - - # If cat_ids is in a 'groups' structure, locate the media group - if self.cat_ids_structure == 'group': - if not group: - raise ValueError("group is required on group cat_ids structure") - - for group_type, group_cat_ids in cat_ids: - if group in toIterable(group_type): - cat_ids = group_cat_ids - - for cats in cat_ids: - ids, qualities = cats + for ids, qualities in self.cat_ids: if identifier in qualities: return ids diff --git a/couchpotato/core/providers/info/base.py b/couchpotato/core/providers/info/base.py index 8d06c322..47b7c000 100644 --- a/couchpotato/core/providers/info/base.py +++ b/couchpotato/core/providers/info/base.py @@ -6,4 +6,12 @@ class MovieProvider(Provider): class ShowProvider(Provider): - type = ['season', 'episode'] + type = 'show' + + +class SeasonProvider(Provider): + type = 'season' + + +class EpisodeProvider(Provider): + type = 'episode' diff --git a/couchpotato/core/providers/torrent/iptorrents/main.py b/couchpotato/core/providers/torrent/iptorrents/main.py index 55fa815f..f030a7e3 100644 --- a/couchpotato/core/providers/torrent/iptorrents/main.py +++ b/couchpotato/core/providers/torrent/iptorrents/main.py @@ -3,7 +3,7 @@ from couchpotato.core.helpers.encoding import tryUrlencode from couchpotato.core.helpers.variable import tryInt from couchpotato.core.logger import CPLog from couchpotato.core.providers.base import MultiProvider -from couchpotato.core.providers.info.base import MovieProvider, ShowProvider +from couchpotato.core.providers.info.base import MovieProvider, SeasonProvider, EpisodeProvider from couchpotato.core.providers.torrent.base import TorrentProvider import traceback @@ -13,7 +13,7 @@ log = CPLog(__name__) class IPTorrents(MultiProvider): def getTypes(self): - return [Movie, Show] + return [Movie, Season, Episode] class Base(TorrentProvider): @@ -29,13 +29,16 @@ class Base(TorrentProvider): http_time_between_calls = 1 #seconds cat_backup_id = None - def _buildUrl(self, query, quality_identifier, cat_ids_group = None): + def buildUrl(self, title, media, quality): + return self._buildUrl(title.replace(':', ''), quality['identifier']) - cat_ids = self.getCatId(quality_identifier, cat_ids_group) + def _buildUrl(self, query, quality_identifier): - if not cat_ids or not len(cat_ids): - log.warning('Unable to find category for quality %s', quality_identifier) - return + cat_ids = self.getCatId(quality_identifier) + + if not cat_ids: + log.warning('Unable to find category ids for identifier "%s"', quality_identifier) + return None return self.urls['search'] % ("&".join(("l%d=" % x) for x in cat_ids), tryUrlencode(query).replace('%', '%%')) @@ -133,20 +136,16 @@ class Movie(MovieProvider, Base): return self._buildUrl(query, quality['identifier']) -class Show(ShowProvider, Base): +class Season(SeasonProvider, Base): cat_ids = [ - ('season', [ - ([65], ['hdtv_sd', 'hdtv_720p', 'webdl_720p', 'webdl_1080p']), - ]), - ('episode', [ - ([5], ['hdtv_720p', 'webdl_720p', 'webdl_1080p']), - ([4, 78, 79], ['hdtv_sd']) - ]) + ([65], ['hdtv_sd', 'hdtv_720p', 'webdl_720p', 'webdl_1080p']), ] - def buildUrl(self, title, media, quality): - if media['type'] not in ['season', 'episode']: - return - return self._buildUrl(title.replace(':', ''), quality['identifier'], media['type']) \ No newline at end of file +class Episode(EpisodeProvider, Base): + + cat_ids = [ + ([5], ['hdtv_720p', 'webdl_720p', 'webdl_1080p']), + ([4, 78, 79], ['hdtv_sd']) + ] diff --git a/couchpotato/core/settings/model.py b/couchpotato/core/settings/model.py index 472f6694..82dbdbe4 100644 --- a/couchpotato/core/settings/model.py +++ b/couchpotato/core/settings/model.py @@ -90,6 +90,7 @@ class Media(Entity): files = ManyToMany('File', cascade = 'all, delete-orphan', single_parent = True) + class Library(Entity): """""" using_options(inheritance = 'multi') @@ -112,6 +113,79 @@ class Library(Entity): parent = ManyToOne('Library') children = OneToMany('Library') + def getRelated(self, include_parents = True, include_self = True, include_children = True, merge=False): + libraries = [] + + if include_parents and self.parent is not None: + libraries += self.parent.getRelated(include_children = False) + + if include_self: + libraries += [(self.type, self)] + + if include_children: + for child in self.children: + libraries += child.getRelated(include_parents = False) + + # Return plain results if we aren't merging the results + if not merge: + return libraries + + # Merge the results into a dict ({type: [,...]}) + root_key = None + results = {} + + for key, library in libraries: + if root_key is None: + root_key = key + + if key not in results: + results[key] = [] + + results[key].append(library) + + return root_key, results + + def to_dict(self, deep = None, exclude = None): + if not exclude: exclude = [] + if not deep: deep = {} + + include_related = False + include_root = False + + if any(x in deep for x in ['related_libraries', 'root_library']): + deep = deep.copy() + + include_related = deep.pop('related_libraries', None) is not None + include_root = deep.pop('root_library', None) is not None + + orig_dict = super(Library, self).to_dict(deep = deep, exclude = exclude) + + # Include related libraries (parents and children) + if include_related: + # Fetch child and parent libraries and determine root type + root_key, related_libraries = self.getRelated(include_self = False, merge=True) + + # Serialize libraries + related_libraries = dict([ + (key, [library.to_dict(deep, exclude) for library in libraries]) + for (key, libraries) in related_libraries.items() + ]) + + # Add a reference to the current library dict into related_libraries + if orig_dict['type'] not in related_libraries: + related_libraries[orig_dict['type']] = [] + + related_libraries[orig_dict['type']].append(orig_dict) + + # Update the dict for this library + orig_dict['related_libraries'] = related_libraries + + if include_root: + root_library = related_libraries.get(root_key) + orig_dict['root_library'] = root_library[0] if len(root_library) else None + + return orig_dict + class ShowLibrary(Library, DictMixin): using_options(inheritance = 'multi') diff --git a/libs/caper/__init__.py b/libs/caper/__init__.py index 1638ec0a..3384d7d7 100644 --- a/libs/caper/__init__.py +++ b/libs/caper/__init__.py @@ -19,7 +19,7 @@ from caper.parsers.anime import AnimeParser from caper.parsers.scene import SceneParser -__version_info__ = ('0', '2', '2') +__version_info__ = ('0', '2', '3') __version_branch__ = 'master' __version__ = "%s%s" % ( @@ -43,10 +43,10 @@ CL_END = 1 class Caper(object): - def __init__(self): + def __init__(self, debug=False): self.parsers = { - 'scene': SceneParser(), - 'anime': AnimeParser() + 'scene': SceneParser(debug), + 'anime': AnimeParser(debug) } def _closure_split(self, name): diff --git a/libs/caper/parsers/anime.py b/libs/caper/parsers/anime.py index 4f90163c..88313a2c 100644 --- a/libs/caper/parsers/anime.py +++ b/libs/caper/parsers/anime.py @@ -53,8 +53,8 @@ PATTERN_GROUPS = [ class AnimeParser(Parser): - def __init__(self): - super(AnimeParser, self).__init__(PATTERN_GROUPS) + def __init__(self, debug=False): + super(AnimeParser, self).__init__(PATTERN_GROUPS, debug) def capture_group(self, fragment): match = REGEX_GROUP.match(fragment.value) diff --git a/libs/caper/parsers/base.py b/libs/caper/parsers/base.py index d0e109e2..6f79be61 100644 --- a/libs/caper/parsers/base.py +++ b/libs/caper/parsers/base.py @@ -18,7 +18,9 @@ from caper.result import CaperResult, CaperClosureNode class Parser(object): - def __init__(self, pattern_groups): + def __init__(self, pattern_groups, debug=False): + self.debug = debug + self.matcher = FragmentMatcher(pattern_groups) self.closures = None diff --git a/libs/caper/parsers/scene.py b/libs/caper/parsers/scene.py index dedd206d..224a282e 100644 --- a/libs/caper/parsers/scene.py +++ b/libs/caper/parsers/scene.py @@ -98,8 +98,8 @@ PATTERN_GROUPS = [ class SceneParser(Parser): - def __init__(self): - super(SceneParser, self).__init__(PATTERN_GROUPS) + def __init__(self, debug=False): + super(SceneParser, self).__init__(PATTERN_GROUPS, debug) def capture_group(self, fragment): if fragment.left_sep == '-' and not fragment.right: @@ -133,6 +133,9 @@ class SceneParser(Parser): return self.result def print_tree(self, heads): + if not self.debug: + return + for head in heads: head = head if type(head) is list else [head] diff --git a/libs/logr/__init__.py b/libs/logr/__init__.py index 21909b61..7a2d7b2e 100644 --- a/libs/logr/__init__.py +++ b/libs/logr/__init__.py @@ -32,8 +32,11 @@ class Logr(object): loggers = {} handler = None + trace_origin = False + name = "Logr" + @staticmethod - def configure(level=logging.WARNING, handler=None, formatter=None): + def configure(level=logging.WARNING, handler=None, formatter=None, trace_origin=False, name="Logr"): """Configure Logr @param handler: Logger message handler @@ -52,6 +55,9 @@ class Logr(object): handler.setLevel(level) Logr.handler = handler + Logr.trace_origin = trace_origin + Logr.name = name + @staticmethod def configure_check(): if Logr.handler is None: @@ -64,8 +70,30 @@ class Logr(object): except TypeError: return "" + @staticmethod + def get_frame_class(frame): + if len(frame.f_code.co_varnames) <= 0: + return None + + farg = frame.f_code.co_varnames[0] + + if farg not in frame.f_locals: + return None + + if farg == 'self': + return frame.f_locals[farg].__class__ + + if farg == 'cls': + return frame.f_locals[farg] + + return None + + @staticmethod def get_logger_name(): + if not Logr.trace_origin: + return Logr.name + stack = inspect.stack() for x in xrange_six(len(stack)): @@ -73,20 +101,16 @@ class Logr(object): name = None # Try find name of function defined inside a class - if len(frame.f_code.co_varnames) > 0: - self_argument = frame.f_code.co_varnames[0] + frame_class = Logr.get_frame_class(frame) - if self_argument == 'self' and self_argument in frame.f_locals: - instance = frame.f_locals[self_argument] + if frame_class: + class_name = frame_class.__name__ + module_name = frame_class.__module__ - class_ = instance.__class__ - class_name = class_.__name__ - module_name = class_.__module__ - - if module_name != '__main__': - name = module_name + '.' + class_name - else: - name = class_name + if module_name != '__main__': + name = module_name + '.' + class_name + else: + name = class_name # Try find name of function defined outside of a class if name is None: