dinislage/CouchPotatoServer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #2549 from fuzeman/tv_searcher

Browse Source 
[TV] Searcher cleanup and matcher updates
This commit is contained in:
Joel Kåberg
2013-12-03 23:16:11 -08:00
parent 9ce8ffc14b 143dcad4f3
commit 180b2bbffe
14 changed files with 243 additions and 154 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
couchpotato/core/media/__init__.py
View File

@@ -1,5 +1,6 @@
from couchpotato import get_session
from couchpotato.core.event import addEvent, fireEventAsync, fireEvent
from couchpotato.core.helpers.variable import mergeDicts
from couchpotato.core.plugins.base import Plugin
from couchpotato.core.settings.model import Media
@@ -17,6 +18,13 @@ class MediaBase(Plugin):
'category': {},
}
search_dict = mergeDicts(default_dict, {
'library': {
'related_libraries': {},
'root_library': {}
},
})
def initType(self):
addEvent('media.types', self.getType)
@@ -28,7 +36,7 @@ class MediaBase(Plugin):
def onComplete():
db = get_session()
media = db.query(Media).filter_by(id = id).first()
fireEventAsync('%s.searcher.single' % media.type, media.to_dict(self.default_dict), on_complete = self.createNotifyFront(id))
fireEventAsync('%s.searcher.single' % media.type, media.to_dict(self.search_dict), on_complete = self.createNotifyFront(id))
db.expire_all()
return onComplete

29
couchpotato/core/media/show/library/episode/main.py
View File

@@ -17,10 +17,39 @@ class EpisodeLibraryPlugin(LibraryBase):
default_dict = {'titles': {}, 'files':{}}
def __init__(self):
addEvent('library.identifier', self.identifier)
addEvent('library.add.episode', self.add)
addEvent('library.update.episode', self.update)
addEvent('library.update.episode_release_date', self.updateReleaseDate)
def identifier(self, library):
if library.get('type') != 'episode':
return
identifier = {
'season': None,
'episode': None
}
scene_map = library['info'].get('map_episode', {}).get('scene')
if scene_map:
# Use scene mappings if they are available
identifier['season'] = scene_map.get('season')
identifier['episode'] = scene_map.get('episode')
else:
# Fallback to normal season/episode numbers
identifier['season'] = library.get('season_number')
identifier['episode'] = library.get('episode_number')
# Cast identifiers to integers
# TODO this will need changing to support identifiers with trailing 'a', 'b' characters
identifier['season'] = tryInt(identifier['season'], None)
identifier['episode'] = tryInt(identifier['episode'], None)
return identifier
def add(self, attrs = {}, update_after = True):
type = attrs.get('type', 'episode')
primary_provider = attrs.get('primary_provider', 'thetvdb')

12
couchpotato/core/media/show/library/season/main.py
View File

@@ -17,10 +17,22 @@ class SeasonLibraryPlugin(LibraryBase):
default_dict = {'titles': {}, 'files':{}}
def __init__(self):
addEvent('library.identifier', self.identifier)
addEvent('library.add.season', self.add)
addEvent('library.update.season', self.update)
addEvent('library.update.season_release_date', self.updateReleaseDate)
def identifier(self, library):
if library.get('type') != 'season':
return
season_num = tryInt(library['season_number'], None)
return {
'season': season_num,
'episode': None
}
def add(self, attrs = {}, update_after = True):
type = attrs.get('type', 'season')
primary_provider = attrs.get('primary_provider', 'thetvdb')

117
couchpotato/core/media/show/searcher/main.py
View File

@@ -1,10 +1,10 @@
from couchpotato import get_session, Env
from couchpotato import Env, get_session
from couchpotato.core.event import addEvent, fireEvent
from couchpotato.core.helpers.variable import getTitle, tryInt
from couchpotato.core.helpers.variable import getTitle, tryInt, toIterable
from couchpotato.core.logger import CPLog
from couchpotato.core.media._base.searcher.main import SearchSetupError
from couchpotato.core.plugins.base import Plugin
from couchpotato.core.settings.model import Media, Library
from couchpotato.core.settings.model import Media
from qcond import QueryCondenser
from qcond.helpers import simplify
@@ -13,6 +13,8 @@ log = CPLog(__name__)
class ShowSearcher(Plugin):
type = ['show', 'season', 'episode']
in_progress = False
# TODO come back to this later, think this could be handled better
@@ -29,16 +31,17 @@ class ShowSearcher(Plugin):
self.query_condenser = QueryCondenser()
addEvent('show.searcher.single', self.single)
for type in toIterable(self.type):
addEvent('%s.searcher.single' % type, self.single)
addEvent('searcher.get_search_title', self.getSearchTitle)
addEvent('searcher.correct_match', self.correctMatch)
addEvent('searcher.correct_release', self.correctRelease)
addEvent('searcher.get_media_identifier', self.getMediaIdentifier)
addEvent('searcher.get_media_root', self.getMediaRoot)
def single(self, media, search_protocols = None, manual = False):
show, season, episode = self.getLibraries(media['library'])
if media['type'] == 'show':
# TODO handle show searches (scan all seasons)
return
@@ -69,8 +72,7 @@ class ShowSearcher(Plugin):
#fireEvent('episode.delete', episode['id'], single = True)
return
show, season, episode = self.getMedia(media)
if show is None or season is None:
if not show or not season:
log.error('Unable to find show or season library in database, missing required data for searching')
return
@@ -93,7 +95,7 @@ class ShowSearcher(Plugin):
# Don't search for quality lower then already available.
if has_better_quality is 0:
log.info('Search for %s S%02d%s in %s', (getTitle(show), season.season_number, "E%02d" % episode.episode_number if episode else "", quality_type['quality']['label']))
log.info('Search for %s S%02d%s in %s', (getTitle(show), season['season_number'], "E%02d" % episode['episode_number'] if episode else "", quality_type['quality']['label']))
quality = fireEvent('quality.single', identifier = quality_type['quality']['identifier'], single = True)
results = fireEvent('searcher.search', search_protocols, media, quality, single = True)
@@ -135,15 +137,16 @@ class ShowSearcher(Plugin):
if media['type'] not in ['show', 'season', 'episode']:
return
show, season, episode = self.getMedia(media)
if show is None:
show, season, episode = self.getLibraries(media['library'])
if not show:
return None
titles = []
# Add season map_names if they exist
if season is not None and 'map_names' in show.info:
season_names = show.info['map_names'].get(str(season.season_number), {})
if season is not None and 'map_names' in show['info']:
season_names = show['info']['map_names'].get(str(season['season_number']), {})
# Add titles from all locations
# TODO only add name maps from a specific location
@@ -151,7 +154,7 @@ class ShowSearcher(Plugin):
titles += [name for name in names if name not in titles]
# Add show titles
titles += [title.title for title in show.titles if title.title not in titles]
titles += [title['title'] for title in show['titles'] if title['title'] not in titles]
# Use QueryCondenser to build a list of optimal search titles
condensed_titles = self.query_condenser.distinct(titles)
@@ -170,9 +173,9 @@ class ShowSearcher(Plugin):
return None
# Add the identifier to search title
# TODO supporting other identifier formats
identifier = fireEvent('searcher.get_media_identifier', media['library'], single = True)
identifier = fireEvent('library.identifier', media['library'], single = True)
# TODO this needs to support other identifier formats
if identifier['season']:
title += ' S%02d' % identifier['season']
@@ -195,11 +198,7 @@ class ShowSearcher(Plugin):
if not fireEvent('searcher.correct_words', release['name'], media, single = True):
return False
show, season, episode = self.getMedia(media)
if show is None or season is None:
log.error('Unable to find show or season library in database, missing required data for searching')
return
# TODO Matching is quite costly, maybe we should be caching release matches somehow? (also look at caper optimizations)
match = fireEvent('matcher.best', release, media, quality, single = True)
if match:
return match.weight
@@ -224,68 +223,24 @@ class ShowSearcher(Plugin):
return True
# TODO move this somewhere else
def getMediaIdentifier(self, media_library):
if media_library['type'] not in ['show', 'season', 'episode']:
return None
def getLibraries(self, library):
if 'related_libraries' not in library:
log.warning("'related_libraries' missing from media library, unable to continue searching")
return None, None, None
identifier = {
'season': None,
'episode': None
}
libraries = library['related_libraries']
if media_library['type'] == 'episode':
map_episode = media_library['info'].get('map_episode')
# Get libraries and return lists only if there is multiple items
show = libraries.get('show', [])
if len(show) <= 1:
show = show[0] if len(show) else None
if map_episode and 'scene' in map_episode:
identifier['season'] = map_episode['scene'].get('season')
identifier['episode'] = map_episode['scene'].get('episode')
else:
# TODO xem mapping?
identifier['season'] = media_library.get('season_number')
identifier['episode'] = media_library.get('episode_number')
season = libraries.get('season', [])
if len(season) <= 1:
season = season[0] if len(season) else None
if media_library['type'] == 'season':
identifier['season'] = media_library.get('season_number')
# Try cast identifier values to integers
identifier['season'] = tryInt(identifier['season'], None)
identifier['episode'] = tryInt(identifier['episode'], None)
return identifier
# TODO move this somewhere else
def getMediaRoot(self, media):
if media['type'] not in ['show', 'season', 'episode']:
return None
show, season, episode = self.getMedia(media)
if show is None or season is None:
log.error('Unable to find show or season library in database, missing required data for searching')
return
return show.to_dict()
# TODO move this somewhere else
def getMedia(self, media):
db = get_session()
media_library = db.query(Library).filter_by(id = media['library_id']).first()
show = None
season = None
episode = None
if media['type'] == 'episode':
show = media_library.parent.parent
season = media_library.parent
episode = media_library
if media['type'] == 'season':
show = media_library.parent
season = media_library
if media['type'] == 'show':
show = media_library
episode = libraries.get('episode', [])
if len(episode) <= 1:
episode = episode[0] if len(episode) else None
return show, season, episode

6
couchpotato/core/plugins/matcher/main.py
View File

@@ -33,7 +33,7 @@ class Matcher(Plugin):
if fireEvent('searcher.correct_match', chain, release, media, quality, single = True):
return chain
return None
return False
def chainMatch(self, chain, group, tags):
found_tags = []
@@ -50,7 +50,7 @@ class Matcher(Plugin):
return set([key for key, value in tags.items() if None not in value]) == set(found_tags)
def correctIdentifier(self, chain, media):
required_id = fireEvent('searcher.get_media_identifier', media['library'], single = True)
required_id = fireEvent('library.identifier', media['library'], single = True)
if 'identifier' not in chain.info:
return False
@@ -73,7 +73,7 @@ class Matcher(Plugin):
return True
def correctTitle(self, chain, media):
root_library = fireEvent('searcher.get_media_root', media['library'], single = True)
root_library = media['library']['root_library']
if 'show_name' not in chain.info or not len(chain.info['show_name']):
log.info('Wrong: missing show name in parsed result')

29
couchpotato/core/providers/base.py
View File

@@ -105,7 +105,6 @@ class YarrProvider(Provider):
type = 'movie'
cat_ids = {}
cat_ids_structure = None
cat_backup_id = None
sizeGb = ['gb', 'gib']
@@ -250,33 +249,9 @@ class YarrProvider(Provider):
return 0
def _discoverCatIdStructure(self):
# Discover cat_ids structure (single or groups)
for group_name, group_cat_ids in self.cat_ids:
if len(group_cat_ids) > 0:
if type(group_cat_ids[0]) is tuple:
self.cat_ids_structure = 'group'
if type(group_cat_ids[0]) is str:
self.cat_ids_structure = 'single'
def getCatId(self, identifier):
def getCatId(self, identifier, group = None):
cat_ids = self.cat_ids
if not self.cat_ids_structure:
self._discoverCatIdStructure()
# If cat_ids is in a 'groups' structure, locate the media group
if self.cat_ids_structure == 'group':
if not group:
raise ValueError("group is required on group cat_ids structure")
for group_type, group_cat_ids in cat_ids:
if group in toIterable(group_type):
cat_ids = group_cat_ids
for cats in cat_ids:
ids, qualities = cats
for ids, qualities in self.cat_ids:
if identifier in qualities:
return ids

10
couchpotato/core/providers/info/base.py
View File

@@ -6,4 +6,12 @@ class MovieProvider(Provider):
class ShowProvider(Provider):
type = ['season', 'episode']
type = 'show'
class SeasonProvider(Provider):
type = 'season'
class EpisodeProvider(Provider):
type = 'episode'

37
couchpotato/core/providers/torrent/iptorrents/main.py
View File

@@ -3,7 +3,7 @@ from couchpotato.core.helpers.encoding import tryUrlencode
from couchpotato.core.helpers.variable import tryInt
from couchpotato.core.logger import CPLog
from couchpotato.core.providers.base import MultiProvider
from couchpotato.core.providers.info.base import MovieProvider, ShowProvider
from couchpotato.core.providers.info.base import MovieProvider, SeasonProvider, EpisodeProvider
from couchpotato.core.providers.torrent.base import TorrentProvider
import traceback
@@ -13,7 +13,7 @@ log = CPLog(__name__)
class IPTorrents(MultiProvider):
def getTypes(self):
return [Movie, Show]
return [Movie, Season, Episode]
class Base(TorrentProvider):
@@ -29,13 +29,16 @@ class Base(TorrentProvider):
http_time_between_calls = 1 #seconds
cat_backup_id = None
def _buildUrl(self, query, quality_identifier, cat_ids_group = None):
def buildUrl(self, title, media, quality):
return self._buildUrl(title.replace(':', ''), quality['identifier'])
cat_ids = self.getCatId(quality_identifier, cat_ids_group)
def _buildUrl(self, query, quality_identifier):
if not cat_ids or not len(cat_ids):
log.warning('Unable to find category for quality %s', quality_identifier)
return
cat_ids = self.getCatId(quality_identifier)
if not cat_ids:
log.warning('Unable to find category ids for identifier "%s"', quality_identifier)
return None
return self.urls['search'] % ("&".join(("l%d=" % x) for x in cat_ids), tryUrlencode(query).replace('%', '%%'))
@@ -133,20 +136,16 @@ class Movie(MovieProvider, Base):
return self._buildUrl(query, quality['identifier'])
class Show(ShowProvider, Base):
class Season(SeasonProvider, Base):
cat_ids = [
('season', [
([65], ['hdtv_sd', 'hdtv_720p', 'webdl_720p', 'webdl_1080p']),
]),
('episode', [
([5], ['hdtv_720p', 'webdl_720p', 'webdl_1080p']),
([4, 78, 79], ['hdtv_sd'])
])
([65], ['hdtv_sd', 'hdtv_720p', 'webdl_720p', 'webdl_1080p']),
]
def buildUrl(self, title, media, quality):
if media['type'] not in ['season', 'episode']:
return
return self._buildUrl(title.replace(':', ''), quality['identifier'], media['type'])
class Episode(EpisodeProvider, Base):
cat_ids = [
([5], ['hdtv_720p', 'webdl_720p', 'webdl_1080p']),
([4, 78, 79], ['hdtv_sd'])
]

74
couchpotato/core/settings/model.py
View File

@@ -90,6 +90,7 @@ class Media(Entity):
files = ManyToMany('File', cascade = 'all, delete-orphan', single_parent = True)
class Library(Entity):
""""""
using_options(inheritance = 'multi')
@@ -112,6 +113,79 @@ class Library(Entity):
parent = ManyToOne('Library')
children = OneToMany('Library')
def getRelated(self, include_parents = True, include_self = True, include_children = True, merge=False):
libraries = []
if include_parents and self.parent is not None:
libraries += self.parent.getRelated(include_children = False)
if include_self:
libraries += [(self.type, self)]
if include_children:
for child in self.children:
libraries += child.getRelated(include_parents = False)
# Return plain results if we aren't merging the results
if not merge:
return libraries
# Merge the results into a dict ({type: [<library>,...]})
root_key = None
results = {}
for key, library in libraries:
if root_key is None:
root_key = key
if key not in results:
results[key] = []
results[key].append(library)
return root_key, results
def to_dict(self, deep = None, exclude = None):
if not exclude: exclude = []
if not deep: deep = {}
include_related = False
include_root = False
if any(x in deep for x in ['related_libraries', 'root_library']):
deep = deep.copy()
include_related = deep.pop('related_libraries', None) is not None
include_root = deep.pop('root_library', None) is not None
orig_dict = super(Library, self).to_dict(deep = deep, exclude = exclude)
# Include related libraries (parents and children)
if include_related:
# Fetch child and parent libraries and determine root type
root_key, related_libraries = self.getRelated(include_self = False, merge=True)
# Serialize libraries
related_libraries = dict([
(key, [library.to_dict(deep, exclude) for library in libraries])
for (key, libraries) in related_libraries.items()
])
# Add a reference to the current library dict into related_libraries
if orig_dict['type'] not in related_libraries:
related_libraries[orig_dict['type']] = []
related_libraries[orig_dict['type']].append(orig_dict)
# Update the dict for this library
orig_dict['related_libraries'] = related_libraries
if include_root:
root_library = related_libraries.get(root_key)
orig_dict['root_library'] = root_library[0] if len(root_library) else None
return orig_dict
class ShowLibrary(Library, DictMixin):
using_options(inheritance = 'multi')

8
libs/caper/__init__.py
View File

@@ -19,7 +19,7 @@ from caper.parsers.anime import AnimeParser
from caper.parsers.scene import SceneParser
__version_info__ = ('0', '2', '2')
__version_info__ = ('0', '2', '3')
__version_branch__ = 'master'
__version__ = "%s%s" % (
@@ -43,10 +43,10 @@ CL_END = 1
class Caper(object):
def __init__(self):
def __init__(self, debug=False):
self.parsers = {
'scene': SceneParser(),
'anime': AnimeParser()
'scene': SceneParser(debug),
'anime': AnimeParser(debug)
}
def _closure_split(self, name):

4
libs/caper/parsers/anime.py
View File

@@ -53,8 +53,8 @@ PATTERN_GROUPS = [
class AnimeParser(Parser):
def __init__(self):
super(AnimeParser, self).__init__(PATTERN_GROUPS)
def __init__(self, debug=False):
super(AnimeParser, self).__init__(PATTERN_GROUPS, debug)
def capture_group(self, fragment):
match = REGEX_GROUP.match(fragment.value)

4
libs/caper/parsers/base.py
View File

@@ -18,7 +18,9 @@ from caper.result import CaperResult, CaperClosureNode
class Parser(object):
def __init__(self, pattern_groups):
def __init__(self, pattern_groups, debug=False):
self.debug = debug
self.matcher = FragmentMatcher(pattern_groups)
self.closures = None

7
libs/caper/parsers/scene.py
View File

@@ -98,8 +98,8 @@ PATTERN_GROUPS = [
class SceneParser(Parser):
def __init__(self):
super(SceneParser, self).__init__(PATTERN_GROUPS)
def __init__(self, debug=False):
super(SceneParser, self).__init__(PATTERN_GROUPS, debug)
def capture_group(self, fragment):
if fragment.left_sep == '-' and not fragment.right:
@@ -133,6 +133,9 @@ class SceneParser(Parser):
return self.result
def print_tree(self, heads):
if not self.debug:
return
for head in heads:
head = head if type(head) is list else [head]

50
libs/logr/__init__.py
View File

@@ -32,8 +32,11 @@ class Logr(object):
loggers = {}
handler = None
trace_origin = False
name = "Logr"
@staticmethod
def configure(level=logging.WARNING, handler=None, formatter=None):
def configure(level=logging.WARNING, handler=None, formatter=None, trace_origin=False, name="Logr"):
"""Configure Logr
@param handler: Logger message handler
@@ -52,6 +55,9 @@ class Logr(object):
handler.setLevel(level)
Logr.handler = handler
Logr.trace_origin = trace_origin
Logr.name = name
@staticmethod
def configure_check():
if Logr.handler is None:
@@ -64,8 +70,30 @@ class Logr(object):
except TypeError:
return "<unknown>"
@staticmethod
def get_frame_class(frame):
if len(frame.f_code.co_varnames) <= 0:
return None
farg = frame.f_code.co_varnames[0]
if farg not in frame.f_locals:
return None
if farg == 'self':
return frame.f_locals[farg].__class__
if farg == 'cls':
return frame.f_locals[farg]
return None
@staticmethod
def get_logger_name():
if not Logr.trace_origin:
return Logr.name
stack = inspect.stack()
for x in xrange_six(len(stack)):
@@ -73,20 +101,16 @@ class Logr(object):
name = None
# Try find name of function defined inside a class
if len(frame.f_code.co_varnames) > 0:
self_argument = frame.f_code.co_varnames[0]
frame_class = Logr.get_frame_class(frame)
if self_argument == 'self' and self_argument in frame.f_locals:
instance = frame.f_locals[self_argument]
if frame_class:
class_name = frame_class.__name__
module_name = frame_class.__module__
class_ = instance.__class__
class_name = class_.__name__
module_name = class_.__module__
if module_name != '__main__':
name = module_name + '.' + class_name
else:
name = class_name
if module_name != '__main__':
name = module_name + '.' + class_name
else:
name = class_name
# Try find name of function defined outside of a class
if name is None: