diff --git a/couchpotato/core/helpers/variable.py b/couchpotato/core/helpers/variable.py index 8c623f63..7d35b997 100644 --- a/couchpotato/core/helpers/variable.py +++ b/couchpotato/core/helpers/variable.py @@ -213,3 +213,6 @@ def randomString(size = 8, chars = string.ascii_uppercase + string.digits): def splitString(str, split_on = ',', clean = True): list = [x.strip() for x in str.split(split_on)] if str else [] return filter(None, list) if clean else list + +def dictIsSubset(a, b): + return all([k in b and b[k] == v for k, v in a.items()]) diff --git a/couchpotato/core/media/_base/searcher/main.py b/couchpotato/core/media/_base/searcher/main.py index f22072c7..b6650114 100644 --- a/couchpotato/core/media/_base/searcher/main.py +++ b/couchpotato/core/media/_base/searcher/main.py @@ -1,18 +1,11 @@ -from couchpotato import get_session from couchpotato.api import addApiView from couchpotato.core.event import addEvent, fireEvent from couchpotato.core.helpers.encoding import simplifyString, toUnicode from couchpotato.core.helpers.variable import md5, getTitle, splitString from couchpotato.core.logger import CPLog from couchpotato.core.media._base.searcher.base import SearcherBase -from couchpotato.core.settings.model import Media, Release, ReleaseInfo -from couchpotato.environment import Env -from inspect import ismethod, isfunction -from sqlalchemy.exc import InterfaceError import datetime import re -import time -import traceback log = CPLog(__name__) @@ -25,9 +18,7 @@ class Searcher(SearcherBase): addEvent('searcher.correct_year', self.correctYear) addEvent('searcher.correct_name', self.correctName) addEvent('searcher.correct_words', self.correctWords) - addEvent('searcher.download', self.download) addEvent('searcher.search', self.search) - addEvent('searcher.create_releases', self.createReleases) addApiView('searcher.full_search', self.searchAllView, docs = { 'desc': 'Starts a full search for all media', @@ -53,93 +44,14 @@ class Searcher(SearcherBase): progress = fireEvent('searcher.progress', merge = True) return progress - def download(self, data, movie, manual = False): - - if not data.get('protocol'): - data['protocol'] = data['type'] - data['type'] = 'movie' - - # Test to see if any downloaders are enabled for this type - downloader_enabled = fireEvent('download.enabled', manual, data, single = True) - - if downloader_enabled: - - snatched_status, done_status, active_status = fireEvent('status.get', ['snatched', 'done', 'active'], single = True) - - # Download movie to temp - filedata = None - if data.get('download') and (ismethod(data.get('download')) or isfunction(data.get('download'))): - filedata = data.get('download')(url = data.get('url'), nzb_id = data.get('id')) - if filedata == 'try_next': - return filedata - - download_result = fireEvent('download', data = data, movie = movie, manual = manual, filedata = filedata, single = True) - log.debug('Downloader result: %s', download_result) - - if download_result: - try: - # Mark release as snatched - db = get_session() - rls = db.query(Release).filter_by(identifier = md5(data['url'])).first() - if rls: - renamer_enabled = Env.setting('enabled', 'renamer') - - # Save download-id info if returned - if isinstance(download_result, dict): - for key in download_result: - rls_info = ReleaseInfo( - identifier = 'download_%s' % key, - value = toUnicode(download_result.get(key)) - ) - rls.info.append(rls_info) - db.commit() - - log_movie = '%s (%s) in %s' % (getTitle(movie['library']), movie['library']['year'], rls.quality.label) - snatch_message = 'Snatched "%s": %s' % (data.get('name'), log_movie) - log.info(snatch_message) - fireEvent('movie.snatched', message = snatch_message, data = rls.to_dict()) - - # If renamer isn't used, mark movie done - if not renamer_enabled: - try: - if movie['status_id'] == active_status.get('id'): - for profile_type in movie['profile']['types']: - if profile_type['quality_id'] == rls.quality.id and profile_type['finish']: - log.info('Renamer disabled, marking movie as finished: %s', log_movie) - - # Mark release done - fireEvent('release.update_status', rls.id, status = done_status, single = True) - - # Mark movie done - mvie = db.query(Media).filter_by(id = movie['id']).first() - mvie.status_id = done_status.get('id') - mvie.last_edit = int(time.time()) - db.commit() - except: - log.error('Failed marking movie finished, renamer disabled: %s', traceback.format_exc()) - else: - fireEvent('release.update_status', rls.id, status = snatched_status, single = True) - - except: - log.error('Failed marking movie finished: %s', traceback.format_exc()) - - return True - - log.info('Tried to download, but none of the "%s" downloaders are enabled or gave an error', (data.get('protocol'))) - - return False - def search(self, protocols, media, quality): results = [] - search_type = None - if media['type'] == 'movie': - search_type = 'movie' - elif media['type'] in ['show', 'season', 'episode']: - search_type = 'show' + # TODO could this be handled better? (removing the need for 'searcher.get_media_searcher_id') + searcher_id = fireEvent('searcher.get_media_searcher_id', media['type'], single = True) for search_protocol in protocols: - protocol_results = fireEvent('provider.search.%s.%s' % (search_protocol, search_type), media, quality, merge = True) + protocol_results = fireEvent('provider.search.%s.%s' % (search_protocol, searcher_id), media, quality, merge = True) if protocol_results: results += protocol_results @@ -151,53 +63,6 @@ class Searcher(SearcherBase): return sorted_results - def createReleases(self, search_results, media, quality_type): - - available_status = fireEvent('status.get', ['available'], single = True) - db = get_session() - - found_releases = [] - - for rel in search_results: - - nzb_identifier = md5(rel['url']) - found_releases.append(nzb_identifier) - - rls = db.query(Release).filter_by(identifier = nzb_identifier).first() - if not rls: - rls = Release( - identifier = nzb_identifier, - movie_id = media.get('id'), - #media_id = media.get('id'), - quality_id = quality_type.get('quality_id'), - status_id = available_status.get('id') - ) - db.add(rls) - else: - [db.delete(old_info) for old_info in rls.info] - rls.last_edit = int(time.time()) - - db.commit() - - for info in rel: - try: - if not isinstance(rel[info], (str, unicode, int, long, float)): - continue - - rls_info = ReleaseInfo( - identifier = info, - value = toUnicode(rel[info]) - ) - rls.info.append(rls_info) - except InterfaceError: - log.debug('Couldn\'t add %s to ReleaseInfo: %s', (info, traceback.format_exc())) - - db.commit() - - rel['status_id'] = rls.status_id - - return found_releases - def getSearchProtocols(self): download_protocols = fireEvent('download.enabled_protocols', merge = True) diff --git a/couchpotato/core/media/movie/_base/static/movie.actions.js b/couchpotato/core/media/movie/_base/static/movie.actions.js index c13cdf10..f6e0f542 100644 --- a/couchpotato/core/media/movie/_base/static/movie.actions.js +++ b/couchpotato/core/media/movie/_base/static/movie.actions.js @@ -393,7 +393,7 @@ MA.Release = new Class({ if(icon) icon.addClass('icon spinner').removeClass('download'); - Api.request('release.download', { + Api.request('release.manual_download', { 'data': { 'id': release.id }, diff --git a/couchpotato/core/media/movie/searcher/main.py b/couchpotato/core/media/movie/searcher/main.py index 09612252..e67bbc52 100644 --- a/couchpotato/core/media/movie/searcher/main.py +++ b/couchpotato/core/media/movie/searcher/main.py @@ -31,6 +31,7 @@ class MovieSearcher(SearcherBase, MovieTypeBase): addEvent('movie.searcher.could_be_released', self.couldBeReleased) addEvent('searcher.correct_release', self.correctRelease) addEvent('searcher.get_search_title', self.getSearchTitle) + addEvent('searcher.get_media_searcher_id', self.getMediaSearcherId) addApiView('movie.searcher.try_next', self.tryNextReleaseView, docs = { 'desc': 'Marks the snatched results as ignored and try the next best release', @@ -179,27 +180,11 @@ class MovieSearcher(SearcherBase, MovieTypeBase): break # Add them to this movie releases list - found_releases += fireEvent('searcher.create_releases', results, movie, quality_type, single = True) + found_releases += fireEvent('release.create_from_search', results, movie, quality_type, single = True) - for nzb in results: - if not quality_type.get('finish', False) and quality_type.get('wait_for', 0) > 0 and nzb.get('age') <= quality_type.get('wait_for', 0): - log.info('Ignored, waiting %s days: %s', (quality_type.get('wait_for'), nzb['name'])) - continue - - if nzb['status_id'] in [ignored_status.get('id'), failed_status.get('id')]: - log.info('Ignored: %s', nzb['name']) - continue - - if nzb['score'] <= 0: - log.info('Ignored, score to low: %s', nzb['name']) - continue - - downloaded = fireEvent('searcher.download', data = nzb, movie = movie, manual = manual, single = True) - if downloaded is True: - ret = True - break - elif downloaded != 'try_next': - break + # Try find a valid result and download it + if fireEvent('release.try_download_result', results, movie, quality_type, manual, single = True): + ret = True # Remove releases that aren't found anymore for release in movie.get('releases', []): @@ -363,5 +348,9 @@ class MovieSearcher(SearcherBase, MovieTypeBase): if media['type'] == 'movie': return getTitle(media['library']) + def getMediaSearcherId(self, media_type): + if media_type == 'movie': + return 'movie' + class SearchSetupError(Exception): pass diff --git a/couchpotato/core/plugins/release/main.py b/couchpotato/core/plugins/release/main.py index 0f885e77..59416030 100644 --- a/couchpotato/core/plugins/release/main.py +++ b/couchpotato/core/plugins/release/main.py @@ -1,11 +1,15 @@ -from couchpotato import get_session +from couchpotato import get_session, md5 from couchpotato.api import addApiView from couchpotato.core.event import fireEvent, addEvent -from couchpotato.core.helpers.encoding import ss +from couchpotato.core.helpers.encoding import ss, toUnicode +from couchpotato.core.helpers.variable import getTitle from couchpotato.core.logger import CPLog from couchpotato.core.plugins.base import Plugin from couchpotato.core.plugins.scanner.main import Scanner -from couchpotato.core.settings.model import File, Release as Relea, Media +from couchpotato.core.settings.model import File, Release as Relea, Media, ReleaseInfo +from couchpotato.environment import Env +from inspect import ismethod, isfunction +from sqlalchemy.exc import InterfaceError from sqlalchemy.orm import joinedload_all from sqlalchemy.sql.expression import and_, or_ import os @@ -20,7 +24,7 @@ class Release(Plugin): def __init__(self): addEvent('release.add', self.add) - addApiView('release.download', self.download, docs = { + addApiView('release.manual_download', self.manualDownload, docs = { 'desc': 'Send a release manually to the downloaders', 'params': { 'id': {'type': 'id', 'desc': 'ID of the release object in release-table'} @@ -45,6 +49,9 @@ class Release(Plugin): } }) + addEvent('release.download', self.download) + addEvent('release.try_download_result', self.tryDownloadResult) + addEvent('release.create_from_search', self.createFromSearch) addEvent('release.for_movie', self.forMovie) addEvent('release.delete', self.delete) addEvent('release.clean', self.clean) @@ -138,7 +145,6 @@ class Release(Plugin): return True - def saveFile(self, filepath, type = 'unknown', include_media_info = False): properties = {} @@ -199,7 +205,7 @@ class Release(Plugin): 'success': True } - def download(self, id = None, **kwargs): + def manualDownload(self, id = None, **kwargs): db = get_session() @@ -211,7 +217,7 @@ class Release(Plugin): for info in rel.info: item[info.identifier] = info.value - fireEvent('notify.frontend', type = 'release.download', data = True, message = 'Snatching "%s"' % item['name']) + fireEvent('notify.frontend', type = 'release.manual_download', data = True, message = 'Snatching "%s"' % item['name']) # Get matching provider provider = fireEvent('provider.belongs_to', item['url'], provider = item.get('provider'), single = True) @@ -223,7 +229,7 @@ class Release(Plugin): if item.get('protocol') != 'torrent_magnet': item['download'] = provider.loginDownload if provider.urls.get('login') else provider.download - success = fireEvent('searcher.download', data = item, movie = rel.movie.to_dict({ + success = fireEvent('release.download', data = item, media = rel.movie.to_dict({ 'profile': {'types': {'quality': {}}}, 'releases': {'status': {}, 'quality': {}}, 'library': {'titles': {}, 'files':{}}, @@ -234,7 +240,7 @@ class Release(Plugin): db.expunge_all() rel = db.query(Relea).filter_by(id = id).first() # Get release again @RuudBurger why do we need to get it again?? - fireEvent('notify.frontend', type = 'release.download', data = True, message = 'Successfully snatched "%s"' % item['name']) + fireEvent('notify.frontend', type = 'release.manual_download', data = True, message = 'Successfully snatched "%s"' % item['name']) return { 'success': success } @@ -245,6 +251,152 @@ class Release(Plugin): 'success': False } + def download(self, data, media, manual = False): + + if not data.get('protocol'): + data['protocol'] = data['type'] + data['type'] = 'movie' + + # Test to see if any downloaders are enabled for this type + downloader_enabled = fireEvent('download.enabled', manual, data, single = True) + + if downloader_enabled: + snatched_status, done_status, active_status = fireEvent('status.get', ['snatched', 'done', 'active'], single = True) + + # Download release to temp + filedata = None + if data.get('download') and (ismethod(data.get('download')) or isfunction(data.get('download'))): + filedata = data.get('download')(url = data.get('url'), nzb_id = data.get('id')) + if filedata == 'try_next': + return filedata + + download_result = fireEvent('download', data = data, movie = media, manual = manual, filedata = filedata, single = True) + log.debug('Downloader result: %s', download_result) + + if download_result: + try: + # Mark release as snatched + db = get_session() + rls = db.query(Relea).filter_by(identifier = md5(data['url'])).first() + if rls: + renamer_enabled = Env.setting('enabled', 'renamer') + + # Save download-id info if returned + if isinstance(download_result, dict): + for key in download_result: + rls_info = ReleaseInfo( + identifier = 'download_%s' % key, + value = toUnicode(download_result.get(key)) + ) + rls.info.append(rls_info) + db.commit() + + log_movie = '%s (%s) in %s' % (getTitle(media['library']), media['library']['year'], rls.quality.label) + snatch_message = 'Snatched "%s": %s' % (data.get('name'), log_movie) + log.info(snatch_message) + fireEvent('%s.snatched' % data['type'], message = snatch_message, data = rls.to_dict()) + + # If renamer isn't used, mark media done + if not renamer_enabled: + try: + if media['status_id'] == active_status.get('id'): + for profile_type in media['profile']['types']: + if profile_type['quality_id'] == rls.quality.id and profile_type['finish']: + log.info('Renamer disabled, marking media as finished: %s', log_movie) + + # Mark release done + fireEvent('release.update_status', rls.id, status = done_status, single = True) + + # Mark media done + mdia = db.query(Media).filter_by(id = media['id']).first() + mdia.status_id = done_status.get('id') + mdia.last_edit = int(time.time()) + db.commit() + except: + log.error('Failed marking media finished, renamer disabled: %s', traceback.format_exc()) + else: + fireEvent('release.update_status', rls.id, status = snatched_status, single = True) + + except: + log.error('Failed marking media finished: %s', traceback.format_exc()) + + return True + + log.info('Tried to download, but none of the "%s" downloaders are enabled or gave an error', (data.get('protocol'))) + + return False + + def tryDownloadResult(self, results, media, quality_type, manual = False): + ignored_status, failed_status = fireEvent('status.get', ['ignored', 'failed'], single = True) + + for rel in results: + if not quality_type.get('finish', False) and quality_type.get('wait_for', 0) > 0 and rel.get('age') <= quality_type.get('wait_for', 0): + log.info('Ignored, waiting %s days: %s', (quality_type.get('wait_for'), rel['name'])) + continue + + if rel['status_id'] in [ignored_status.get('id'), failed_status.get('id')]: + log.info('Ignored: %s', rel['name']) + continue + + if rel['score'] <= 0: + log.info('Ignored, score to low: %s', rel['name']) + continue + + downloaded = fireEvent('release.download', data = rel, media = media, manual = manual, single = True) + if downloaded is True: + return True + elif downloaded != 'try_next': + break + + return False + + def createFromSearch(self, search_results, media, quality_type): + + available_status = fireEvent('status.get', ['available'], single = True) + db = get_session() + + found_releases = [] + + for rel in search_results: + + rel_identifier = md5(rel['url']) + found_releases.append(rel_identifier) + + rls = db.query(Relea).filter_by(identifier = rel_identifier).first() + if not rls: + rls = Relea( + identifier = rel_identifier, + movie_id = media.get('id'), + #media_id = media.get('id'), + quality_id = quality_type.get('quality_id'), + status_id = available_status.get('id') + ) + db.add(rls) + else: + [db.delete(old_info) for old_info in rls.info] + rls.last_edit = int(time.time()) + + db.commit() + + for info in rel: + try: + if not isinstance(rel[info], (str, unicode, int, long, float)): + continue + + rls_info = ReleaseInfo( + identifier = info, + value = toUnicode(rel[info]) + ) + rls.info.append(rls_info) + except InterfaceError: + log.debug('Couldn\'t add %s to ReleaseInfo: %s', (info, traceback.format_exc())) + + db.commit() + + rel['status_id'] = rls.status_id + + return found_releases + def forMovie(self, id = None): db = get_session() diff --git a/couchpotato/core/providers/base.py b/couchpotato/core/providers/base.py index 60420a13..7226cec7 100644 --- a/couchpotato/core/providers/base.py +++ b/couchpotato/core/providers/base.py @@ -256,7 +256,10 @@ class YarrProvider(Provider): if identifier in qualities: return ids - return [self.cat_backup_id] + if self.cat_backup_id: + return [self.cat_backup_id] + + return [] class ResultList(list): @@ -284,12 +287,23 @@ class ResultList(list): new_result = self.fillResult(result) - is_correct_movie = fireEvent('searcher.correct_release', new_result, self.movie, self.quality, + is_correct = fireEvent('searcher.correct_release', new_result, self.movie, self.quality, imdb_results = self.kwargs.get('imdb_results', False), single = True) - if is_correct_movie and new_result['id'] not in self.result_ids: + if is_correct and new_result['id'] not in self.result_ids: + is_correct_weight = float(is_correct) + new_result['score'] += fireEvent('score.calculate', new_result, self.movie, single = True) + old_score = new_result['score'] + new_result['score'] = int(old_score * is_correct_weight) + + log.info('Found correct release with weight %.02f, old_score(%d) now scaled to score(%d)', ( + is_correct_weight, + old_score, + new_result['score'] + )) + self.found(new_result) self.result_ids.append(result['id'])