diff --git a/couchpotato/core/helpers/variable.py b/couchpotato/core/helpers/variable.py index d93c9417..15f9936d 100644 --- a/couchpotato/core/helpers/variable.py +++ b/couchpotato/core/helpers/variable.py @@ -1,3 +1,4 @@ +import collections from couchpotato.core.helpers.encoding import simplifyString, toSafeString, ss from couchpotato.core.logger import CPLog import hashlib @@ -145,9 +146,9 @@ def getImdb(txt, check_inside = False, multiple = False): return False -def tryInt(s): +def tryInt(s, default=0): try: return int(s) - except: return 0 + except: return default def tryFloat(s): try: @@ -163,6 +164,11 @@ def natsortKey(s): def natcmp(a, b): return cmp(natsortKey(a), natsortKey(b)) +def toIterable(value): + if isinstance(value, collections.Iterable): + return value + return [value] + def getTitle(library_dict): try: try: diff --git a/couchpotato/core/media/_base/searcher/main.py b/couchpotato/core/media/_base/searcher/main.py index b6b36125..a96f8452 100644 --- a/couchpotato/core/media/_base/searcher/main.py +++ b/couchpotato/core/media/_base/searcher/main.py @@ -2,11 +2,12 @@ from couchpotato import get_session from couchpotato.api import addApiView from couchpotato.core.event import addEvent, fireEvent from couchpotato.core.helpers.encoding import simplifyString, toUnicode -from couchpotato.core.helpers.variable import md5, getTitle +from couchpotato.core.helpers.variable import md5, getTitle, splitString from couchpotato.core.logger import CPLog from couchpotato.core.media._base.searcher.base import SearcherBase from couchpotato.core.settings.model import Media, Release, ReleaseInfo from couchpotato.environment import Env +from sqlalchemy.exc import InterfaceError from inspect import ismethod, isfunction import datetime import re @@ -23,7 +24,10 @@ class Searcher(SearcherBase): addEvent('searcher.contains_other_quality', self.containsOtherQuality) addEvent('searcher.correct_year', self.correctYear) addEvent('searcher.correct_name', self.correctName) + addEvent('searcher.correct_words', self.correctWords) addEvent('searcher.download', self.download) + addEvent('searcher.search', self.search) + addEvent('searcher.create_releases', self.createReleases) addApiView('searcher.full_search', self.searchAllView, docs = { 'desc': 'Starts a full search for all media', @@ -60,7 +64,7 @@ class Searcher(SearcherBase): if downloader_enabled: - snatched_status, done_status, active_status = fireEvent('status.get', ['snatched', 'done', 'active'], single = True) + snatched_status = fireEvent('status.get', 'snatched', single = True) # Download movie to temp filedata = None @@ -79,7 +83,9 @@ class Searcher(SearcherBase): rls = db.query(Release).filter_by(identifier = md5(data['url'])).first() if rls: renamer_enabled = Env.setting('enabled', 'renamer') - fireEvent('release.update_status', rls.id, status = done_status if not renamer_enabled else snatched_status, single = True) + + done_status = fireEvent('status.get', 'done', single = True) + rls.status_id = done_status.get('id') if not renamer_enabled else snatched_status.get('id') # Save download-id info if returned if isinstance(download_result, dict): @@ -98,12 +104,20 @@ class Searcher(SearcherBase): # If renamer isn't used, mark movie done if not renamer_enabled: + active_status = fireEvent('status.get', 'active', single = True) + done_status = fireEvent('status.get', 'done', single = True) try: if movie['status_id'] == active_status.get('id'): for profile_type in movie['profile']['types']: if profile_type['quality_id'] == rls.quality.id and profile_type['finish']: - # Mark movie done log.info('Renamer disabled, marking movie as finished: %s', log_movie) + + # Mark release done + rls.status_id = done_status.get('id') + rls.last_edit = int(time.time()) + db.commit() + + # Mark movie done mvie = db.query(Media).filter_by(id = movie['id']).first() mvie.status_id = done_status.get('id') mvie.last_edit = int(time.time()) @@ -120,6 +134,75 @@ class Searcher(SearcherBase): return False + def search(self, protocols, media, quality): + results = [] + + search_type = None + if media['type'] == 'movie': + search_type = 'movie' + elif media['type'] in ['show', 'season', 'episode']: + search_type = 'show' + + for search_protocol in protocols: + protocol_results = fireEvent('provider.search.%s.%s' % (search_protocol, search_type), media, quality, merge = True) + if protocol_results: + results += protocol_results + + sorted_results = sorted(results, key = lambda k: k['score'], reverse = True) + + download_preference = self.conf('preferred_method', section = 'searcher') + if download_preference != 'both': + sorted_results = sorted(sorted_results, key = lambda k: k['protocol'][:3], reverse = (download_preference == 'torrent')) + + return sorted_results + + def createReleases(self, search_results, media, quality_type): + + available_status, ignored_status, failed_status = fireEvent('status.get', ['available', 'ignored', 'failed'], single = True) + db = get_session() + + found_releases = [] + + for rel in search_results: + + nzb_identifier = md5(rel['url']) + found_releases.append(nzb_identifier) + + rls = db.query(Release).filter_by(identifier = nzb_identifier).first() + if not rls: + rls = Release( + identifier = nzb_identifier, + movie_id = media.get('id'), + #media_id = media.get('id'), + quality_id = quality_type.get('quality_id'), + status_id = available_status.get('id') + ) + db.add(rls) + else: + [db.delete(old_info) for old_info in rls.info] + rls.last_edit = int(time.time()) + + db.commit() + + for info in rel: + try: + if not isinstance(rel[info], (str, unicode, int, long, float)): + continue + + rls_info = ReleaseInfo( + identifier = info, + value = toUnicode(rel[info]) + ) + rls.info.append(rls_info) + except InterfaceError: + log.debug('Couldn\'t add %s to ReleaseInfo: %s', (info, traceback.format_exc())) + + db.commit() + + rel['status_id'] = rls.status_id + + return found_releases + def getSearchProtocols(self): download_protocols = fireEvent('download.enabled_protocols', merge = True) @@ -224,5 +307,49 @@ class Searcher(SearcherBase): return False + def correctWords(self, rel_name, media): + media_title = fireEvent('searcher.get_search_title', media, single = True) + media_words = re.split('\W+', simplifyString(media_title)) + + rel_name = simplifyString(rel_name) + rel_words = re.split('\W+', rel_name) + + # Make sure it has required words + required_words = splitString(self.conf('required_words', section = 'searcher').lower()) + try: required_words = list(set(required_words + splitString(media['category']['required'].lower()))) + except: pass + + req_match = 0 + for req_set in required_words: + req = splitString(req_set, '&') + req_match += len(list(set(rel_words) & set(req))) == len(req) + + if len(required_words) > 0 and req_match == 0: + log.info2('Wrong: Required word missing: %s', rel_name) + return False + + # Ignore releases + ignored_words = splitString(self.conf('ignored_words', section = 'searcher').lower()) + try: ignored_words = list(set(ignored_words + splitString(media['category']['ignored'].lower()))) + except: pass + + ignored_match = 0 + for ignored_set in ignored_words: + ignored = splitString(ignored_set, '&') + ignored_match += len(list(set(rel_words) & set(ignored))) == len(ignored) + + if len(ignored_words) > 0 and ignored_match: + log.info2("Wrong: '%s' contains 'ignored words'", rel_name) + return False + + # Ignore porn stuff + pron_tags = ['xxx', 'sex', 'anal', 'tits', 'fuck', 'porn', 'orgy', 'milf', 'boobs', 'erotica', 'erotic', 'cock', 'dick'] + pron_words = list(set(rel_words) & set(pron_tags) - set(media_words)) + if pron_words: + log.info('Wrong: %s, probably pr0n', rel_name) + return False + + return True + class SearchSetupError(Exception): pass diff --git a/couchpotato/core/media/movie/searcher/main.py b/couchpotato/core/media/movie/searcher/main.py index 79d2ed84..63dcf761 100644 --- a/couchpotato/core/media/movie/searcher/main.py +++ b/couchpotato/core/media/movie/searcher/main.py @@ -1,16 +1,14 @@ from couchpotato import get_session from couchpotato.api import addApiView from couchpotato.core.event import addEvent, fireEvent, fireEventAsync -from couchpotato.core.helpers.encoding import simplifyString, toUnicode, ss -from couchpotato.core.helpers.variable import md5, getTitle, splitString, \ - possibleTitles, getImdb +from couchpotato.core.helpers.encoding import simplifyString +from couchpotato.core.helpers.variable import getTitle, possibleTitles, getImdb from couchpotato.core.logger import CPLog from couchpotato.core.media._base.searcher.base import SearcherBase from couchpotato.core.media.movie import MovieTypeBase -from couchpotato.core.settings.model import Media, Release, ReleaseInfo +from couchpotato.core.settings.model import Media, Release from couchpotato.environment import Env from datetime import date -from sqlalchemy.exc import InterfaceError import random import re import time @@ -29,9 +27,10 @@ class MovieSearcher(SearcherBase, MovieTypeBase): addEvent('movie.searcher.all', self.searchAll) addEvent('movie.searcher.all_view', self.searchAllView) addEvent('movie.searcher.single', self.single) - addEvent('movie.searcher.correct_movie', self.correctMovie) addEvent('movie.searcher.try_next_release', self.tryNextRelease) addEvent('movie.searcher.could_be_released', self.couldBeReleased) + addEvent('searcher.correct_release', self.correctRelease) + addEvent('searcher.get_search_title', self.getSearchTitle) addApiView('movie.searcher.try_next', self.tryNextReleaseView, docs = { 'desc': 'Marks the snatched results as ignored and try the next best release', @@ -117,6 +116,10 @@ class MovieSearcher(SearcherBase, MovieTypeBase): def single(self, movie, search_protocols = None, manual = False): + # movies don't contain 'type' yet, so just set to default here + if 'type' not in movie: + movie['type'] = 'movie' + # Find out search type try: if not search_protocols: @@ -167,64 +170,18 @@ class MovieSearcher(SearcherBase, MovieTypeBase): log.info('Search for %s in %s', (default_title, quality_type['quality']['label'])) quality = fireEvent('quality.single', identifier = quality_type['quality']['identifier'], single = True) - results = [] - for search_protocol in search_protocols: - protocol_results = fireEvent('provider.search.%s.movie' % search_protocol, movie, quality, merge = True) - if protocol_results: - results += protocol_results - - sorted_results = sorted(results, key = lambda k: k['score'], reverse = True) - if len(sorted_results) == 0: + results = fireEvent('searcher.search', search_protocols, movie, quality, single = True) + if len(results) == 0: log.debug('Nothing found for %s in %s', (default_title, quality_type['quality']['label'])) - download_preference = self.conf('preferred_method', section = 'searcher') - if download_preference != 'both': - sorted_results = sorted(sorted_results, key = lambda k: k['protocol'][:3], reverse = (download_preference == 'torrent')) - # Check if movie isn't deleted while searching if not db.query(Media).filter_by(id = movie.get('id')).first(): break # Add them to this movie releases list - for nzb in sorted_results: + found_releases += fireEvent('searcher.create_releases', results, movie, quality_type, single = True) - nzb_identifier = md5(nzb['url']) - found_releases.append(nzb_identifier) - - rls = db.query(Release).filter_by(identifier = nzb_identifier).first() - if not rls: - rls = Release( - identifier = nzb_identifier, - movie_id = movie.get('id'), - quality_id = quality_type.get('quality_id'), - status_id = available_status.get('id') - ) - db.add(rls) - else: - [db.delete(old_info) for old_info in rls.info] - rls.last_edit = int(time.time()) - - db.commit() - - for info in nzb: - try: - if not isinstance(nzb[info], (str, unicode, int, long, float)): - continue - - rls_info = ReleaseInfo( - identifier = info, - value = toUnicode(nzb[info]) - ) - rls.info.append(rls_info) - except InterfaceError: - log.debug('Couldn\'t add %s to ReleaseInfo: %s', (info, traceback.format_exc())) - - db.commit() - - nzb['status_id'] = rls.status_id - - - for nzb in sorted_results: + for nzb in results: if not quality_type.get('finish', False) and quality_type.get('wait_for', 0) > 0 and nzb.get('age') <= quality_type.get('wait_for', 0): log.info('Ignored, waiting %s days: %s', (quality_type.get('wait_for'), nzb['name'])) continue @@ -265,7 +222,11 @@ class MovieSearcher(SearcherBase, MovieTypeBase): return ret - def correctMovie(self, nzb = None, movie = None, quality = None, **kwargs): + def correctRelease(self, nzb = None, media = None, quality = None, **kwargs): + + if media.get('type') != 'movie': return + + media_title = fireEvent('searcher.get_search_title', media, single = True) imdb_results = kwargs.get('imdb_results', False) retention = Env.setting('retention', section = 'nzb') @@ -274,50 +235,14 @@ class MovieSearcher(SearcherBase, MovieTypeBase): log.info2('Wrong: Outside retention, age is %s, needs %s or lower: %s', (nzb['age'], retention, nzb['name'])) return False - movie_name = getTitle(movie['library']) - movie_words = re.split('\W+', simplifyString(movie_name)) - nzb_name = simplifyString(nzb['name']) - nzb_words = re.split('\W+', nzb_name) - - # Make sure it has required words - required_words = splitString(self.conf('required_words', section = 'searcher').lower()) - try: required_words = list(set(required_words + splitString(movie['category']['required'].lower()))) - except: pass - - req_match = 0 - for req_set in required_words: - req = splitString(req_set, '&') - req_match += len(list(set(nzb_words) & set(req))) == len(req) - - if len(required_words) > 0 and req_match == 0: - log.info2('Wrong: Required word missing: %s', nzb['name']) - return False - - # Ignore releases - ignored_words = splitString(self.conf('ignored_words', section = 'searcher').lower()) - try: ignored_words = list(set(ignored_words + splitString(movie['category']['ignored'].lower()))) - except: pass - - ignored_match = 0 - for ignored_set in ignored_words: - ignored = splitString(ignored_set, '&') - ignored_match += len(list(set(nzb_words) & set(ignored))) == len(ignored) - - if len(ignored_words) > 0 and ignored_match: - log.info2("Wrong: '%s' contains 'ignored words'", (nzb['name'])) - return False - - # Ignore porn stuff - pron_tags = ['xxx', 'sex', 'anal', 'tits', 'fuck', 'porn', 'orgy', 'milf', 'boobs', 'erotica', 'erotic', 'cock', 'dick'] - pron_words = list(set(nzb_words) & set(pron_tags) - set(movie_words)) - if pron_words: - log.info('Wrong: %s, probably pr0n', (nzb['name'])) + # Check for required and ignored words + if not fireEvent('searcher.correct_words', nzb['name'], media, single = True): return False preferred_quality = fireEvent('quality.single', identifier = quality['identifier'], single = True) # Contains lower quality string - if fireEvent('searcher.contains_other_quality', nzb, movie_year = movie['library']['year'], preferred_quality = preferred_quality, single = True): + if fireEvent('searcher.contains_other_quality', nzb, movie_year = media['library']['year'], preferred_quality = preferred_quality, single = True): log.info2('Wrong: %s, looking for %s', (nzb['name'], quality['label'])) return False @@ -347,23 +272,23 @@ class MovieSearcher(SearcherBase, MovieTypeBase): return True # Check if nzb contains imdb link - if getImdb(nzb.get('description', '')) == movie['library']['identifier']: + if getImdb(nzb.get('description', '')) == media['library']['identifier']: return True - for raw_title in movie['library']['titles']: + for raw_title in media['library']['titles']: for movie_title in possibleTitles(raw_title['title']): movie_words = re.split('\W+', simplifyString(movie_title)) if fireEvent('searcher.correct_name', nzb['name'], movie_title, single = True): # if no IMDB link, at least check year range 1 - if len(movie_words) > 2 and fireEvent('searcher.correct_year', nzb['name'], movie['library']['year'], 1, single = True): + if len(movie_words) > 2 and fireEvent('searcher.correct_year', nzb['name'], media['library']['year'], 1, single = True): return True # if no IMDB link, at least check year - if len(movie_words) <= 2 and fireEvent('searcher.correct_year', nzb['name'], movie['library']['year'], 0, single = True): + if len(movie_words) <= 2 and fireEvent('searcher.correct_year', nzb['name'], media['library']['year'], 0, single = True): return True - log.info("Wrong: %s, undetermined naming. Looking for '%s (%s)'", (nzb['name'], movie_name, movie['library']['year'])) + log.info("Wrong: %s, undetermined naming. Looking for '%s (%s)'", (nzb['name'], media_title, media['library']['year'])) return False def couldBeReleased(self, is_pre_release, dates, year = None): @@ -434,5 +359,9 @@ class MovieSearcher(SearcherBase, MovieTypeBase): log.error('Failed searching for next release: %s', traceback.format_exc()) return False + def getSearchTitle(self, media): + if media['type'] == 'movie': + return getTitle(media['library']) + class SearchSetupError(Exception): pass diff --git a/couchpotato/core/providers/base.py b/couchpotato/core/providers/base.py index e6a9cb00..2fa8c11d 100644 --- a/couchpotato/core/providers/base.py +++ b/couchpotato/core/providers/base.py @@ -1,3 +1,4 @@ +import logging from couchpotato.core.event import addEvent, fireEvent from couchpotato.core.helpers.variable import tryFloat, mergeDicts, md5, \ possibleTitles, getTitle @@ -15,7 +16,6 @@ import xml.etree.ElementTree as XMLTree log = CPLog(__name__) - class MultiProvider(Plugin): def __init__(self): @@ -279,8 +279,7 @@ class ResultList(list): new_result = self.fillResult(result) - is_correct_movie = fireEvent('movie.searcher.correct_movie', - nzb = new_result, movie = self.movie, quality = self.quality, + is_correct_movie = fireEvent('searcher.correct_release', new_result, self.movie, self.quality, imdb_results = self.kwargs.get('imdb_results', False), single = True) if is_correct_movie and new_result['id'] not in self.result_ids: