From dd5ae3c4ee1c07cc896a2a16c4814ce2e4d5a989 Mon Sep 17 00:00:00 2001 From: Dean Gardiner Date: Mon, 30 Sep 2013 22:03:21 +1300 Subject: [PATCH] Working TV correctRelease function with quality, identifier and show title checking. --- couchpotato/core/helpers/variable.py | 4 +- couchpotato/core/media/show/searcher/main.py | 138 ++++++++++++++++-- couchpotato/core/plugins/quality/main.py | 10 +- .../core/providers/torrent/iptorrents/main.py | 23 +-- 4 files changed, 152 insertions(+), 23 deletions(-) diff --git a/couchpotato/core/helpers/variable.py b/couchpotato/core/helpers/variable.py index 77b110a4..15f9936d 100644 --- a/couchpotato/core/helpers/variable.py +++ b/couchpotato/core/helpers/variable.py @@ -146,9 +146,9 @@ def getImdb(txt, check_inside = False, multiple = False): return False -def tryInt(s): +def tryInt(s, default=0): try: return int(s) - except: return 0 + except: return default def tryFloat(s): try: diff --git a/couchpotato/core/media/show/searcher/main.py b/couchpotato/core/media/show/searcher/main.py index 1f851552..ae4316ff 100644 --- a/couchpotato/core/media/show/searcher/main.py +++ b/couchpotato/core/media/show/searcher/main.py @@ -1,11 +1,14 @@ import pprint +import re from couchpotato import get_session, Env from couchpotato.core.event import addEvent, fireEvent -from couchpotato.core.helpers.variable import getTitle +from couchpotato.core.helpers.encoding import simplifyString +from couchpotato.core.helpers.variable import getTitle, tryInt, possibleTitles from couchpotato.core.logger import CPLog from couchpotato.core.media._base.searcher.main import SearchSetupError from couchpotato.core.plugins.base import Plugin -from couchpotato.core.settings.model import Media +from couchpotato.core.settings.model import Media, Library +from caper import Caper log = CPLog(__name__) @@ -14,6 +17,15 @@ class ShowSearcher(Plugin): in_progress = False + # TODO come back to this later, think this could be handled better + quality_map = { + 'webdl_1080p': {'resolution': ['1080p'], 'source': ['webdl']}, + 'webdl_720p': {'resolution': ['720p'], 'source': ['webdl']}, + + 'hdtv_720p': {'resolution': ['720p'], 'source': ['hdtv']}, + 'hdtv_sd': {'resolution': ['480p', None], 'source': ['hdtv']}, + } + def __init__(self): super(ShowSearcher, self).__init__() @@ -21,10 +33,12 @@ class ShowSearcher(Plugin): addEvent('searcher.correct_release', self.correctRelease) addEvent('searcher.get_search_title', self.getSearchTitle) + self.caper = Caper() + def _lookupMedia(self, media): db = get_session() - media_library = db.query(Media).filter_by(id = media['id']).first().library + media_library = db.query(Library).filter_by(id = media['library_id']).first() show = None season = None @@ -45,6 +59,8 @@ class ShowSearcher(Plugin): return show, season, episode def single(self, media, search_protocols = None): + pprint.pprint(media) + if media['type'] == 'show': # TODO handle show searches (scan all seasons) return @@ -70,7 +86,7 @@ class ShowSearcher(Plugin): found_releases = [] too_early_to_search = [] - default_title = getTitle(media['library']) + default_title = self.getSearchTitle(media['library']) if not default_title: log.error('No proper info found for episode, removing it from library to cause it from having more issues.') #fireEvent('episode.delete', episode['id'], single = True) @@ -126,14 +142,118 @@ class ShowSearcher(Plugin): if not fireEvent('searcher.correct_words', release['name'], media, single = True): return False - preferred_quality = fireEvent('quality.single', identifier = quality['identifier'], single = True) + #pprint.pprint(release) - # Contains lower quality string - if fireEvent('searcher.contains_other_quality', release, preferred_quality = preferred_quality, single = True): - log.info2('Wrong: %s, looking for %s', (release['name'], quality['label'])) + show, season, episode = self._lookupMedia(media) + if show is None or season is None: + log.error('Unable to find show or season library in database, missing required data for searching') + return + + release_info = self.caper.parse(release['name']) + if len(release_info.chains) < 1: + log.info2('Wrong: %s, unable to parse release name (no chains)', release['name']) return False - pprint.pprint(release) + # TODO look at all chains + chain = release_info.chains[0] + + if not self.correctQuality(chain, quality['identifier']): + log.info('Wrong: %s, quality does not match', release['name']) + return False + + if not self.correctIdentifier(chain, media): + log.info('Wrong: %s, identifier does not match', release['name']) + return False + + #print chain.weight + #pprint.pprint(chain.info) + + if 'show_name' not in chain.info or not len(chain.info['show_name']): + log.info('Wrong: %s, missing show name in parsed result', release['name']) + return False + + chain_words = [x.lower() for x in chain.info['show_name']] + chain_title = ' '.join(chain_words) + + library_title = None + + # Check show titles match + for raw_title in show.titles: + for valid_words in [x.split(' ') for x in possibleTitles(raw_title.title)]: + if not library_title: + library_title = ' '.join(valid_words) + + if valid_words == chain_words: + return True + + log.info("Wrong: title '%s', undetermined show naming. Looking for '%s (%s)'", (chain_title, library_title, media['library']['year'])) + return False + + def correctQuality(self, chain, quality_identifier): + if quality_identifier not in self.quality_map: + log.info2('Wrong: unknown preferred quality %s for TV searching', quality_identifier) + return False + + if 'video' not in chain.info: + log.info2('Wrong: no video tags found') + return False + + video_tags = self.quality_map[quality_identifier] + + if not self.chainMatches(chain, 'video', video_tags): + log.info2('Wrong: %s tags not in chain', video_tags) + return False + + return True + + def correctIdentifier(self, chain, media): + required_id = self.getIdentifier(media['library'], 'season_number', 'episode_number') + + if 'identifier' not in chain.info: + return False + + # TODO could be handled better? + if len(chain.info['identifier']) != 1: + return False + identifier = chain.info['identifier'][0] + + # TODO air by date episodes + release_id = self.getIdentifier(identifier, 'season', 'episode') + + if required_id != release_id: + log.info2('Wrong: required identifier %s does not match release identifier %s', (str(required_id), str(release_id))) + return False + + return True + + def getIdentifier(self, d, episode_key, season_key): + return ( + tryInt(d.get(season_key), None) if season_key in d else None, + tryInt(d.get(episode_key), None) if episode_key in d else None + ) + + def chainMatches(self, chain, group, tags): + found_tags = [] + + for match in chain.info[group]: + for ck, cv in match.items(): + if ck in tags and self.cleanMatchValue(cv) in tags[ck]: + found_tags.append(ck) + + + if set(tags.keys()) == set(found_tags): + return True + + return set([key for key, value in tags.items() if value]) == set(found_tags) + + def cleanMatchValue(self, value): + value = value.lower() + value = value.strip() + + for ch in [' ', '-', '.']: + value = value.replace(ch, '') + + return value def getSearchTitle(self, media): show, season, episode = self._lookupMedia(media) diff --git a/couchpotato/core/plugins/quality/main.py b/couchpotato/core/plugins/quality/main.py index 1149c036..c853ca74 100644 --- a/couchpotato/core/plugins/quality/main.py +++ b/couchpotato/core/plugins/quality/main.py @@ -26,7 +26,15 @@ class QualityPlugin(Plugin): {'identifier': 'r5', 'size': (600, 1000), 'label': 'R5', 'alternative': ['r6'], 'allow': ['dvdr'], 'ext':['avi', 'mpg', 'mpeg']}, {'identifier': 'tc', 'size': (600, 1000), 'label': 'TeleCine', 'alternative': ['telecine'], 'allow': [], 'ext':['avi', 'mpg', 'mpeg']}, {'identifier': 'ts', 'size': (600, 1000), 'label': 'TeleSync', 'alternative': ['telesync', 'hdts'], 'allow': [], 'ext':['avi', 'mpg', 'mpeg']}, - {'identifier': 'cam', 'size': (600, 1000), 'label': 'Cam', 'alternative': ['camrip', 'hdcam'], 'allow': [], 'ext':['avi', 'mpg', 'mpeg']} + {'identifier': 'cam', 'size': (600, 1000), 'label': 'Cam', 'alternative': ['camrip', 'hdcam'], 'allow': [], 'ext':['avi', 'mpg', 'mpeg']}, + + # TODO come back to this later, think this could be handled better + # WEB-DL + {'identifier': 'webdl_1080p', 'hd': True, 'size': (800, 5000), 'label': 'WEB-DL - 1080p', 'width': 1920, 'height': 1080, 'alternative': [], 'allow': [], 'ext':['mkv']}, + {'identifier': 'webdl_720p', 'hd': True, 'size': (800, 5000), 'label': 'WEB-DL - 720p', 'width': 1280, 'height': 720, 'alternative': [], 'allow': [], 'ext':['mkv']}, + # HDTV + {'identifier': 'hdtv_720p', 'hd': True, 'size': (800, 5000), 'label': 'HDTV - 720p', 'width': 1280, 'height': 720, 'alternative': [], 'allow': [], 'ext':['mkv']}, + {'identifier': 'hdtv_sd', 'hd': False, 'size': (100, 1000), 'label': 'HDTV - SD', 'width': 720, 'alternative': [], 'allow': [], 'ext':['mkv', 'mp4', 'avi']}, ] pre_releases = ['cam', 'ts', 'tc', 'r5', 'scr'] diff --git a/couchpotato/core/providers/torrent/iptorrents/main.py b/couchpotato/core/providers/torrent/iptorrents/main.py index da3ccf48..e8247c54 100644 --- a/couchpotato/core/providers/torrent/iptorrents/main.py +++ b/couchpotato/core/providers/torrent/iptorrents/main.py @@ -36,21 +36,22 @@ class Base(TorrentProvider): log.warning('Unable to find category for quality %s', quality_identifier) return - return self.urls['search'] % (cat_id, tryUrlencode(query)) + return self.urls['search'] % (cat_id, tryUrlencode(query).replace('%', '%%')) def _searchOnTitle(self, title, media, quality, results): freeleech = '' if not self.conf('freeleech') else '&free=on' + base_url = self.buildUrl(title, media, quality) + if not base_url: return + pages = 1 current_page = 1 while current_page <= pages and not self.shuttingDown(): - url = self.buildUrl(title, media, quality) - if not url: return - - url = url % (freeleech, current_page) - - data = self.getHTMLData(url, opener = self.login_opener) + data = self.getHTMLData( + base_url % (freeleech, current_page), + opener = self.login_opener + ) if data: html = BeautifulSoup(data) @@ -135,12 +136,12 @@ class Show(ShowProvider, Base): cat_ids = [ ('season', [ - ([65], ['hdtv', '480p', '720p', '1080p']), + ([65], ['hdtv_sd', 'hdtv_720p', 'webdl_720p', 'webdl_1080p']), ]), ('episode', [ - ([5], ['720p', '1080p']), - ([78], ['480p']), - ([4, 79], ['hdtv']) + ([5], ['hdtv_720p', 'webdl_720p', 'webdl_1080p']), + ([78], ['hdtv_sd']), + ([4, 79], ['hdtv_sd']) ]) ]