Merge branch 'refs/heads/fuzeman-dev_searcher' into develop

couchpotato/core/helpers/variable.py

@@ -1,3 +1,4 @@
+import collections
 from couchpotato.core.helpers.encoding import simplifyString, toSafeString, ss
 from couchpotato.core.logger import CPLog
 import hashlib
@@ -145,9 +146,9 @@ def getImdb(txt, check_inside = False, multiple = False):
 
     return False
 
-def tryInt(s):
+def tryInt(s, default=0):
     try: return int(s)
-    except: return 0
+    except: return default
 
 def tryFloat(s):
     try:
@@ -163,6 +164,11 @@ def natsortKey(s):
 def natcmp(a, b):
     return cmp(natsortKey(a), natsortKey(b))
 
+def toIterable(value):
+    if isinstance(value, collections.Iterable):
+        return value
+    return [value]
+
 def getTitle(library_dict):
     try:
         try:

couchpotato/core/media/_base/searcher/main.py

@@ -2,11 +2,12 @@ from couchpotato import get_session
 from couchpotato.api import addApiView
 from couchpotato.core.event import addEvent, fireEvent
 from couchpotato.core.helpers.encoding import simplifyString, toUnicode
-from couchpotato.core.helpers.variable import md5, getTitle
+from couchpotato.core.helpers.variable import md5, getTitle, splitString
 from couchpotato.core.logger import CPLog
 from couchpotato.core.media._base.searcher.base import SearcherBase
 from couchpotato.core.settings.model import Media, Release, ReleaseInfo
 from couchpotato.environment import Env
+from sqlalchemy.exc import InterfaceError
 from inspect import ismethod, isfunction
 import datetime
 import re
@@ -23,7 +24,10 @@ class Searcher(SearcherBase):
         addEvent('searcher.contains_other_quality', self.containsOtherQuality)
         addEvent('searcher.correct_year', self.correctYear)
         addEvent('searcher.correct_name', self.correctName)
+        addEvent('searcher.correct_words', self.correctWords)
         addEvent('searcher.download', self.download)
+        addEvent('searcher.search', self.search)
+        addEvent('searcher.create_releases', self.createReleases)
 
         addApiView('searcher.full_search', self.searchAllView, docs = {
             'desc': 'Starts a full search for all media',
@@ -60,7 +64,7 @@ class Searcher(SearcherBase):
 
         if downloader_enabled:
 
-            snatched_status, done_status, active_status = fireEvent('status.get', ['snatched', 'done', 'active'], single = True)
+            snatched_status = fireEvent('status.get', 'snatched', single = True)
 
             # Download movie to temp
             filedata = None
@@ -79,7 +83,9 @@ class Searcher(SearcherBase):
                     rls = db.query(Release).filter_by(identifier = md5(data['url'])).first()
                     if rls:
                         renamer_enabled = Env.setting('enabled', 'renamer')
-                        fireEvent('release.update_status', rls.id, status = done_status if not renamer_enabled else snatched_status, single = True)
+
+                        done_status = fireEvent('status.get', 'done', single = True)
+                        rls.status_id = done_status.get('id') if not renamer_enabled else snatched_status.get('id')
 
                         # Save download-id info if returned
                         if isinstance(download_result, dict):
@@ -98,12 +104,20 @@ class Searcher(SearcherBase):
 
                         # If renamer isn't used, mark movie done
                         if not renamer_enabled:
+                            active_status = fireEvent('status.get', 'active', single = True)
+                            done_status = fireEvent('status.get', 'done', single = True)
                             try:
                                 if movie['status_id'] == active_status.get('id'):
                                     for profile_type in movie['profile']['types']:
                                         if profile_type['quality_id'] == rls.quality.id and profile_type['finish']:
-                                            # Mark movie done
                                             log.info('Renamer disabled, marking movie as finished: %s', log_movie)
+
+                                            # Mark release done
+                                            rls.status_id = done_status.get('id')
+                                            rls.last_edit = int(time.time())
+                                            db.commit()
+
+                                            # Mark movie done
                                             mvie = db.query(Media).filter_by(id = movie['id']).first()
                                             mvie.status_id = done_status.get('id')
                                             mvie.last_edit = int(time.time())
@@ -120,6 +134,75 @@ class Searcher(SearcherBase):
 
         return False
 
+    def search(self, protocols, media, quality):
+        results = []
+
+        search_type = None
+        if media['type'] == 'movie':
+            search_type = 'movie'
+        elif media['type'] in ['show', 'season', 'episode']:
+            search_type = 'show'
+
+        for search_protocol in protocols:
+            protocol_results = fireEvent('provider.search.%s.%s' % (search_protocol, search_type), media, quality, merge = True)
+            if protocol_results:
+                results += protocol_results
+
+        sorted_results = sorted(results, key = lambda k: k['score'], reverse = True)
+
+        download_preference = self.conf('preferred_method', section = 'searcher')
+        if download_preference != 'both':
+            sorted_results = sorted(sorted_results, key = lambda k: k['protocol'][:3], reverse = (download_preference == 'torrent'))
+
+        return sorted_results
+
+    def createReleases(self, search_results, media, quality_type):
+
+        available_status, ignored_status, failed_status = fireEvent('status.get', ['available', 'ignored', 'failed'], single = True)
+        db = get_session()
+
+        found_releases = []
+
+        for rel in search_results:
+
+            nzb_identifier = md5(rel['url'])
+            found_releases.append(nzb_identifier)
+
+            rls = db.query(Release).filter_by(identifier = nzb_identifier).first()
+            if not rls:
+                rls = Release(
+                    identifier = nzb_identifier,
+                    movie_id = media.get('id'),
+                    #media_id = media.get('id'),
+                    quality_id = quality_type.get('quality_id'),
+                    status_id = available_status.get('id')
+                )
+                db.add(rls)
+            else:
+                [db.delete(old_info) for old_info in rls.info]
+                rls.last_edit = int(time.time())
+
+            db.commit()
+
+            for info in rel:
+                try:
+                    if not isinstance(rel[info], (str, unicode, int, long, float)):
+                        continue
+
+                    rls_info = ReleaseInfo(
+                        identifier = info,
+                        value = toUnicode(rel[info])
+                    )
+                    rls.info.append(rls_info)
+                except InterfaceError:
+                    log.debug('Couldn\'t add %s to ReleaseInfo: %s', (info, traceback.format_exc()))
+
+            db.commit()
+
+            rel['status_id'] = rls.status_id
+
+        return found_releases
+
     def getSearchProtocols(self):
 
         download_protocols = fireEvent('download.enabled_protocols', merge = True)
@@ -224,5 +307,49 @@ class Searcher(SearcherBase):
 
         return False
 
+    def correctWords(self, rel_name, media):
+        media_title = fireEvent('searcher.get_search_title', media, single = True)
+        media_words = re.split('\W+', simplifyString(media_title))
+
+        rel_name = simplifyString(rel_name)
+        rel_words = re.split('\W+', rel_name)
+
+        # Make sure it has required words
+        required_words = splitString(self.conf('required_words', section = 'searcher').lower())
+        try: required_words = list(set(required_words + splitString(media['category']['required'].lower())))
+        except: pass
+
+        req_match = 0
+        for req_set in required_words:
+            req = splitString(req_set, '&')
+            req_match += len(list(set(rel_words) & set(req))) == len(req)
+
+        if len(required_words) > 0  and req_match == 0:
+            log.info2('Wrong: Required word missing: %s', rel_name)
+            return False
+
+        # Ignore releases
+        ignored_words = splitString(self.conf('ignored_words', section = 'searcher').lower())
+        try: ignored_words = list(set(ignored_words + splitString(media['category']['ignored'].lower())))
+        except: pass
+
+        ignored_match = 0
+        for ignored_set in ignored_words:
+            ignored = splitString(ignored_set, '&')
+            ignored_match += len(list(set(rel_words) & set(ignored))) == len(ignored)
+
+        if len(ignored_words) > 0 and ignored_match:
+            log.info2("Wrong: '%s' contains 'ignored words'", rel_name)
+            return False
+
+        # Ignore porn stuff
+        pron_tags = ['xxx', 'sex', 'anal', 'tits', 'fuck', 'porn', 'orgy', 'milf', 'boobs', 'erotica', 'erotic', 'cock', 'dick']
+        pron_words = list(set(rel_words) & set(pron_tags) - set(media_words))
+        if pron_words:
+            log.info('Wrong: %s, probably pr0n', rel_name)
+            return False
+
+        return True
+
 class SearchSetupError(Exception):
     pass

couchpotato/core/media/movie/searcher/main.py

@@ -1,16 +1,14 @@
 from couchpotato import get_session
 from couchpotato.api import addApiView
 from couchpotato.core.event import addEvent, fireEvent, fireEventAsync
-from couchpotato.core.helpers.encoding import simplifyString, toUnicode, ss
+from couchpotato.core.helpers.encoding import simplifyString
-from couchpotato.core.helpers.variable import md5, getTitle, splitString, \
+from couchpotato.core.helpers.variable import getTitle, possibleTitles, getImdb
-    possibleTitles, getImdb
 from couchpotato.core.logger import CPLog
 from couchpotato.core.media._base.searcher.base import SearcherBase
 from couchpotato.core.media.movie import MovieTypeBase
-from couchpotato.core.settings.model import Media, Release, ReleaseInfo
+from couchpotato.core.settings.model import Media, Release
 from couchpotato.environment import Env
 from datetime import date
-from sqlalchemy.exc import InterfaceError
 import random
 import re
 import time
@@ -29,9 +27,10 @@ class MovieSearcher(SearcherBase, MovieTypeBase):
         addEvent('movie.searcher.all', self.searchAll)
         addEvent('movie.searcher.all_view', self.searchAllView)
         addEvent('movie.searcher.single', self.single)
-        addEvent('movie.searcher.correct_movie', self.correctMovie)
         addEvent('movie.searcher.try_next_release', self.tryNextRelease)
         addEvent('movie.searcher.could_be_released', self.couldBeReleased)
+        addEvent('searcher.correct_release', self.correctRelease)
+        addEvent('searcher.get_search_title', self.getSearchTitle)
 
         addApiView('movie.searcher.try_next', self.tryNextReleaseView, docs = {
             'desc': 'Marks the snatched results as ignored and try the next best release',
@@ -117,6 +116,10 @@ class MovieSearcher(SearcherBase, MovieTypeBase):
 
     def single(self, movie, search_protocols = None, manual = False):
 
+        # movies don't contain 'type' yet, so just set to default here
+        if 'type' not in movie:
+            movie['type'] = 'movie'
+
         # Find out search type
         try:
             if not search_protocols:
@@ -167,64 +170,18 @@ class MovieSearcher(SearcherBase, MovieTypeBase):
                 log.info('Search for %s in %s', (default_title, quality_type['quality']['label']))
                 quality = fireEvent('quality.single', identifier = quality_type['quality']['identifier'], single = True)
 
-                results = []
+                results = fireEvent('searcher.search', search_protocols, movie, quality, single = True)
-                for search_protocol in search_protocols:
+                if len(results) == 0:
-                    protocol_results = fireEvent('provider.search.%s.movie' % search_protocol, movie, quality, merge = True)
-                    if protocol_results:
-                        results += protocol_results
-
-                sorted_results = sorted(results, key = lambda k: k['score'], reverse = True)
-                if len(sorted_results) == 0:
                     log.debug('Nothing found for %s in %s', (default_title, quality_type['quality']['label']))
 
-                download_preference = self.conf('preferred_method', section = 'searcher')
-                if download_preference != 'both':
-                    sorted_results = sorted(sorted_results, key = lambda k: k['protocol'][:3], reverse = (download_preference == 'torrent'))
-
                 # Check if movie isn't deleted while searching
                 if not db.query(Media).filter_by(id = movie.get('id')).first():
                     break
 
                 # Add them to this movie releases list
-                for nzb in sorted_results:
+                found_releases += fireEvent('searcher.create_releases', results, movie, quality_type, single = True)
 
-                    nzb_identifier = md5(nzb['url'])
+                for nzb in results:
-                    found_releases.append(nzb_identifier)
-
-                    rls = db.query(Release).filter_by(identifier = nzb_identifier).first()
-                    if not rls:
-                        rls = Release(
-                            identifier = nzb_identifier,
-                            movie_id = movie.get('id'),
-                            quality_id = quality_type.get('quality_id'),
-                            status_id = available_status.get('id')
-                        )
-                        db.add(rls)
-                    else:
-                        [db.delete(old_info) for old_info in rls.info]
-                        rls.last_edit = int(time.time())
-
-                    db.commit()
-
-                    for info in nzb:
-                        try:
-                            if not isinstance(nzb[info], (str, unicode, int, long, float)):
-                                continue
-
-                            rls_info = ReleaseInfo(
-                                identifier = info,
-                                value = toUnicode(nzb[info])
-                            )
-                            rls.info.append(rls_info)
-                        except InterfaceError:
-                            log.debug('Couldn\'t add %s to ReleaseInfo: %s', (info, traceback.format_exc()))
-
-                    db.commit()
-
-                    nzb['status_id'] = rls.status_id
-
-
-                for nzb in sorted_results:
                     if not quality_type.get('finish', False) and quality_type.get('wait_for', 0) > 0 and nzb.get('age') <= quality_type.get('wait_for', 0):
                         log.info('Ignored, waiting %s days: %s', (quality_type.get('wait_for'), nzb['name']))
                         continue
@@ -265,7 +222,11 @@ class MovieSearcher(SearcherBase, MovieTypeBase):
 
         return ret
 
-    def correctMovie(self, nzb = None, movie = None, quality = None, **kwargs):
+    def correctRelease(self, nzb = None, media = None, quality = None, **kwargs):
+
+        if media.get('type') != 'movie': return
+
+        media_title = fireEvent('searcher.get_search_title', media, single = True)
 
         imdb_results = kwargs.get('imdb_results', False)
         retention = Env.setting('retention', section = 'nzb')
@@ -274,50 +235,14 @@ class MovieSearcher(SearcherBase, MovieTypeBase):
             log.info2('Wrong: Outside retention, age is %s, needs %s or lower: %s', (nzb['age'], retention, nzb['name']))
             return False
 
-        movie_name = getTitle(movie['library'])
+        # Check for required and ignored words
-        movie_words = re.split('\W+', simplifyString(movie_name))
+        if not fireEvent('searcher.correct_words', nzb['name'], media, single = True):
-        nzb_name = simplifyString(nzb['name'])
-        nzb_words = re.split('\W+', nzb_name)
-
-        # Make sure it has required words
-        required_words = splitString(self.conf('required_words', section = 'searcher').lower())
-        try: required_words = list(set(required_words + splitString(movie['category']['required'].lower())))
-        except: pass
-
-        req_match = 0
-        for req_set in required_words:
-            req = splitString(req_set, '&')
-            req_match += len(list(set(nzb_words) & set(req))) == len(req)
-
-        if len(required_words) > 0  and req_match == 0:
-            log.info2('Wrong: Required word missing: %s', nzb['name'])
-            return False
-
-        # Ignore releases
-        ignored_words = splitString(self.conf('ignored_words', section = 'searcher').lower())
-        try: ignored_words = list(set(ignored_words + splitString(movie['category']['ignored'].lower())))
-        except: pass
-
-        ignored_match = 0
-        for ignored_set in ignored_words:
-            ignored = splitString(ignored_set, '&')
-            ignored_match += len(list(set(nzb_words) & set(ignored))) == len(ignored)
-
-        if len(ignored_words) > 0 and ignored_match:
-            log.info2("Wrong: '%s' contains 'ignored words'", (nzb['name']))
-            return False
-
-        # Ignore porn stuff
-        pron_tags = ['xxx', 'sex', 'anal', 'tits', 'fuck', 'porn', 'orgy', 'milf', 'boobs', 'erotica', 'erotic', 'cock', 'dick']
-        pron_words = list(set(nzb_words) & set(pron_tags) - set(movie_words))
-        if pron_words:
-            log.info('Wrong: %s, probably pr0n', (nzb['name']))
             return False
 
         preferred_quality = fireEvent('quality.single', identifier = quality['identifier'], single = True)
 
         # Contains lower quality string
-        if fireEvent('searcher.contains_other_quality', nzb, movie_year = movie['library']['year'], preferred_quality = preferred_quality, single = True):
+        if fireEvent('searcher.contains_other_quality', nzb, movie_year = media['library']['year'], preferred_quality = preferred_quality, single = True):
             log.info2('Wrong: %s, looking for %s', (nzb['name'], quality['label']))
             return False
 
@@ -347,23 +272,23 @@ class MovieSearcher(SearcherBase, MovieTypeBase):
             return True
 
         # Check if nzb contains imdb link
-        if getImdb(nzb.get('description', '')) == movie['library']['identifier']:
+        if getImdb(nzb.get('description', '')) == media['library']['identifier']:
             return True
 
-        for raw_title in movie['library']['titles']:
+        for raw_title in media['library']['titles']:
             for movie_title in possibleTitles(raw_title['title']):
                 movie_words = re.split('\W+', simplifyString(movie_title))
 
                 if fireEvent('searcher.correct_name', nzb['name'], movie_title, single = True):
                     # if no IMDB link, at least check year range 1
-                    if len(movie_words) > 2 and fireEvent('searcher.correct_year', nzb['name'], movie['library']['year'], 1, single = True):
+                    if len(movie_words) > 2 and fireEvent('searcher.correct_year', nzb['name'], media['library']['year'], 1, single = True):
                         return True
 
                     # if no IMDB link, at least check year
-                    if len(movie_words) <= 2 and fireEvent('searcher.correct_year', nzb['name'], movie['library']['year'], 0, single = True):
+                    if len(movie_words) <= 2 and fireEvent('searcher.correct_year', nzb['name'], media['library']['year'], 0, single = True):
                         return True
 
-        log.info("Wrong: %s, undetermined naming. Looking for '%s (%s)'", (nzb['name'], movie_name, movie['library']['year']))
+        log.info("Wrong: %s, undetermined naming. Looking for '%s (%s)'", (nzb['name'], media_title, media['library']['year']))
         return False
 
     def couldBeReleased(self, is_pre_release, dates, year = None):
@@ -434,5 +359,9 @@ class MovieSearcher(SearcherBase, MovieTypeBase):
             log.error('Failed searching for next release: %s', traceback.format_exc())
             return False
 
+    def getSearchTitle(self, media):
+        if media['type'] == 'movie':
+            return getTitle(media['library'])
+
 class SearchSetupError(Exception):
     pass

couchpotato/core/providers/base.py

@@ -1,3 +1,4 @@
+import logging
 from couchpotato.core.event import addEvent, fireEvent
 from couchpotato.core.helpers.variable import tryFloat, mergeDicts, md5, \
     possibleTitles, getTitle
@@ -15,7 +16,6 @@ import xml.etree.ElementTree as XMLTree
 
 log = CPLog(__name__)
 
-
 class MultiProvider(Plugin):
 
     def __init__(self):
@@ -279,8 +279,7 @@ class ResultList(list):
 
         new_result = self.fillResult(result)
 
-        is_correct_movie = fireEvent('movie.searcher.correct_movie',
+        is_correct_movie = fireEvent('searcher.correct_release', new_result, self.movie, self.quality,
-                                     nzb = new_result, movie = self.movie, quality = self.quality,
                                      imdb_results = self.kwargs.get('imdb_results', False), single = True)
 
         if is_correct_movie and new_result['id'] not in self.result_ids: