Merge branch 'refs/heads/fuzeman-dev_searcher' into develop
This commit is contained in:
Ruud
@@ -213,3 +213,6 @@ def randomString(size = 8, chars = string.ascii_uppercase + string.digits):
|
||||
def splitString(str, split_on = ',', clean = True):
|
||||
list = [x.strip() for x in str.split(split_on)] if str else []
|
||||
return filter(None, list) if clean else list
|
||||
|
||||
def dictIsSubset(a, b):
|
||||
return all([k in b and b[k] == v for k, v in a.items()])
|
||||
|
||||
@@ -1,18 +1,11 @@
|
||||
from couchpotato import get_session
|
||||
from couchpotato.api import addApiView
|
||||
from couchpotato.core.event import addEvent, fireEvent
|
||||
from couchpotato.core.helpers.encoding import simplifyString, toUnicode
|
||||
from couchpotato.core.helpers.variable import md5, getTitle, splitString
|
||||
from couchpotato.core.logger import CPLog
|
||||
from couchpotato.core.media._base.searcher.base import SearcherBase
|
||||
from couchpotato.core.settings.model import Media, Release, ReleaseInfo
|
||||
from couchpotato.environment import Env
|
||||
from inspect import ismethod, isfunction
|
||||
from sqlalchemy.exc import InterfaceError
|
||||
import datetime
|
||||
import re
|
||||
import time
|
||||
import traceback
|
||||
|
||||
log = CPLog(__name__)
|
||||
|
||||
@@ -25,9 +18,7 @@ class Searcher(SearcherBase):
|
||||
addEvent('searcher.correct_year', self.correctYear)
|
||||
addEvent('searcher.correct_name', self.correctName)
|
||||
addEvent('searcher.correct_words', self.correctWords)
|
||||
addEvent('searcher.download', self.download)
|
||||
addEvent('searcher.search', self.search)
|
||||
addEvent('searcher.create_releases', self.createReleases)
|
||||
|
||||
addApiView('searcher.full_search', self.searchAllView, docs = {
|
||||
'desc': 'Starts a full search for all media',
|
||||
@@ -53,93 +44,14 @@ class Searcher(SearcherBase):
|
||||
progress = fireEvent('searcher.progress', merge = True)
|
||||
return progress
|
||||
|
||||
def download(self, data, movie, manual = False):
|
||||
|
||||
if not data.get('protocol'):
|
||||
data['protocol'] = data['type']
|
||||
data['type'] = 'movie'
|
||||
|
||||
# Test to see if any downloaders are enabled for this type
|
||||
downloader_enabled = fireEvent('download.enabled', manual, data, single = True)
|
||||
|
||||
if downloader_enabled:
|
||||
|
||||
snatched_status, done_status, active_status = fireEvent('status.get', ['snatched', 'done', 'active'], single = True)
|
||||
|
||||
# Download movie to temp
|
||||
filedata = None
|
||||
if data.get('download') and (ismethod(data.get('download')) or isfunction(data.get('download'))):
|
||||
filedata = data.get('download')(url = data.get('url'), nzb_id = data.get('id'))
|
||||
if filedata == 'try_next':
|
||||
return filedata
|
||||
|
||||
download_result = fireEvent('download', data = data, movie = movie, manual = manual, filedata = filedata, single = True)
|
||||
log.debug('Downloader result: %s', download_result)
|
||||
|
||||
if download_result:
|
||||
try:
|
||||
# Mark release as snatched
|
||||
db = get_session()
|
||||
rls = db.query(Release).filter_by(identifier = md5(data['url'])).first()
|
||||
if rls:
|
||||
renamer_enabled = Env.setting('enabled', 'renamer')
|
||||
|
||||
# Save download-id info if returned
|
||||
if isinstance(download_result, dict):
|
||||
for key in download_result:
|
||||
rls_info = ReleaseInfo(
|
||||
identifier = 'download_%s' % key,
|
||||
value = toUnicode(download_result.get(key))
|
||||
)
|
||||
rls.info.append(rls_info)
|
||||
db.commit()
|
||||
|
||||
log_movie = '%s (%s) in %s' % (getTitle(movie['library']), movie['library']['year'], rls.quality.label)
|
||||
snatch_message = 'Snatched "%s": %s' % (data.get('name'), log_movie)
|
||||
log.info(snatch_message)
|
||||
fireEvent('movie.snatched', message = snatch_message, data = rls.to_dict())
|
||||
|
||||
# If renamer isn't used, mark movie done
|
||||
if not renamer_enabled:
|
||||
try:
|
||||
if movie['status_id'] == active_status.get('id'):
|
||||
for profile_type in movie['profile']['types']:
|
||||
if profile_type['quality_id'] == rls.quality.id and profile_type['finish']:
|
||||
log.info('Renamer disabled, marking movie as finished: %s', log_movie)
|
||||
|
||||
# Mark release done
|
||||
fireEvent('release.update_status', rls.id, status = done_status, single = True)
|
||||
|
||||
# Mark movie done
|
||||
mvie = db.query(Media).filter_by(id = movie['id']).first()
|
||||
mvie.status_id = done_status.get('id')
|
||||
mvie.last_edit = int(time.time())
|
||||
db.commit()
|
||||
except:
|
||||
log.error('Failed marking movie finished, renamer disabled: %s', traceback.format_exc())
|
||||
else:
|
||||
fireEvent('release.update_status', rls.id, status = snatched_status, single = True)
|
||||
|
||||
except:
|
||||
log.error('Failed marking movie finished: %s', traceback.format_exc())
|
||||
|
||||
return True
|
||||
|
||||
log.info('Tried to download, but none of the "%s" downloaders are enabled or gave an error', (data.get('protocol')))
|
||||
|
||||
return False
|
||||
|
||||
def search(self, protocols, media, quality):
|
||||
results = []
|
||||
|
||||
search_type = None
|
||||
if media['type'] == 'movie':
|
||||
search_type = 'movie'
|
||||
elif media['type'] in ['show', 'season', 'episode']:
|
||||
search_type = 'show'
|
||||
# TODO could this be handled better? (removing the need for 'searcher.get_media_searcher_id')
|
||||
searcher_id = fireEvent('searcher.get_media_searcher_id', media['type'], single = True)
|
||||
|
||||
for search_protocol in protocols:
|
||||
protocol_results = fireEvent('provider.search.%s.%s' % (search_protocol, search_type), media, quality, merge = True)
|
||||
protocol_results = fireEvent('provider.search.%s.%s' % (search_protocol, searcher_id), media, quality, merge = True)
|
||||
if protocol_results:
|
||||
results += protocol_results
|
||||
|
||||
@@ -151,53 +63,6 @@ class Searcher(SearcherBase):
|
||||
|
||||
return sorted_results
|
||||
|
||||
def createReleases(self, search_results, media, quality_type):
|
||||
|
||||
available_status = fireEvent('status.get', ['available'], single = True)
|
||||
db = get_session()
|
||||
|
||||
found_releases = []
|
||||
|
||||
for rel in search_results:
|
||||
|
||||
nzb_identifier = md5(rel['url'])
|
||||
found_releases.append(nzb_identifier)
|
||||
|
||||
rls = db.query(Release).filter_by(identifier = nzb_identifier).first()
|
||||
if not rls:
|
||||
rls = Release(
|
||||
identifier = nzb_identifier,
|
||||
movie_id = media.get('id'),
|
||||
#media_id = media.get('id'),
|
||||
quality_id = quality_type.get('quality_id'),
|
||||
status_id = available_status.get('id')
|
||||
)
|
||||
db.add(rls)
|
||||
else:
|
||||
[db.delete(old_info) for old_info in rls.info]
|
||||
rls.last_edit = int(time.time())
|
||||
|
||||
db.commit()
|
||||
|
||||
for info in rel:
|
||||
try:
|
||||
if not isinstance(rel[info], (str, unicode, int, long, float)):
|
||||
continue
|
||||
|
||||
rls_info = ReleaseInfo(
|
||||
identifier = info,
|
||||
value = toUnicode(rel[info])
|
||||
)
|
||||
rls.info.append(rls_info)
|
||||
except InterfaceError:
|
||||
log.debug('Couldn\'t add %s to ReleaseInfo: %s', (info, traceback.format_exc()))
|
||||
|
||||
db.commit()
|
||||
|
||||
rel['status_id'] = rls.status_id
|
||||
|
||||
return found_releases
|
||||
|
||||
def getSearchProtocols(self):
|
||||
|
||||
download_protocols = fireEvent('download.enabled_protocols', merge = True)
|
||||
|
||||
@@ -393,7 +393,7 @@ MA.Release = new Class({
|
||||
if(icon)
|
||||
icon.addClass('icon spinner').removeClass('download');
|
||||
|
||||
Api.request('release.download', {
|
||||
Api.request('release.manual_download', {
|
||||
'data': {
|
||||
'id': release.id
|
||||
},
|
||||
|
||||
@@ -31,6 +31,7 @@ class MovieSearcher(SearcherBase, MovieTypeBase):
|
||||
addEvent('movie.searcher.could_be_released', self.couldBeReleased)
|
||||
addEvent('searcher.correct_release', self.correctRelease)
|
||||
addEvent('searcher.get_search_title', self.getSearchTitle)
|
||||
addEvent('searcher.get_media_searcher_id', self.getMediaSearcherId)
|
||||
|
||||
addApiView('movie.searcher.try_next', self.tryNextReleaseView, docs = {
|
||||
'desc': 'Marks the snatched results as ignored and try the next best release',
|
||||
@@ -179,27 +180,11 @@ class MovieSearcher(SearcherBase, MovieTypeBase):
|
||||
break
|
||||
|
||||
# Add them to this movie releases list
|
||||
found_releases += fireEvent('searcher.create_releases', results, movie, quality_type, single = True)
|
||||
found_releases += fireEvent('release.create_from_search', results, movie, quality_type, single = True)
|
||||
|
||||
for nzb in results:
|
||||
if not quality_type.get('finish', False) and quality_type.get('wait_for', 0) > 0 and nzb.get('age') <= quality_type.get('wait_for', 0):
|
||||
log.info('Ignored, waiting %s days: %s', (quality_type.get('wait_for'), nzb['name']))
|
||||
continue
|
||||
|
||||
if nzb['status_id'] in [ignored_status.get('id'), failed_status.get('id')]:
|
||||
log.info('Ignored: %s', nzb['name'])
|
||||
continue
|
||||
|
||||
if nzb['score'] <= 0:
|
||||
log.info('Ignored, score to low: %s', nzb['name'])
|
||||
continue
|
||||
|
||||
downloaded = fireEvent('searcher.download', data = nzb, movie = movie, manual = manual, single = True)
|
||||
if downloaded is True:
|
||||
ret = True
|
||||
break
|
||||
elif downloaded != 'try_next':
|
||||
break
|
||||
# Try find a valid result and download it
|
||||
if fireEvent('release.try_download_result', results, movie, quality_type, manual, single = True):
|
||||
ret = True
|
||||
|
||||
# Remove releases that aren't found anymore
|
||||
for release in movie.get('releases', []):
|
||||
@@ -363,5 +348,9 @@ class MovieSearcher(SearcherBase, MovieTypeBase):
|
||||
if media['type'] == 'movie':
|
||||
return getTitle(media['library'])
|
||||
|
||||
def getMediaSearcherId(self, media_type):
|
||||
if media_type == 'movie':
|
||||
return 'movie'
|
||||
|
||||
class SearchSetupError(Exception):
|
||||
pass
|
||||
|
||||
@@ -1,11 +1,15 @@
|
||||
from couchpotato import get_session
|
||||
from couchpotato import get_session, md5
|
||||
from couchpotato.api import addApiView
|
||||
from couchpotato.core.event import fireEvent, addEvent
|
||||
from couchpotato.core.helpers.encoding import ss
|
||||
from couchpotato.core.helpers.encoding import ss, toUnicode
|
||||
from couchpotato.core.helpers.variable import getTitle
|
||||
from couchpotato.core.logger import CPLog
|
||||
from couchpotato.core.plugins.base import Plugin
|
||||
from couchpotato.core.plugins.scanner.main import Scanner
|
||||
from couchpotato.core.settings.model import File, Release as Relea, Media
|
||||
from couchpotato.core.settings.model import File, Release as Relea, Media, ReleaseInfo
|
||||
from couchpotato.environment import Env
|
||||
from inspect import ismethod, isfunction
|
||||
from sqlalchemy.exc import InterfaceError
|
||||
from sqlalchemy.orm import joinedload_all
|
||||
from sqlalchemy.sql.expression import and_, or_
|
||||
import os
|
||||
@@ -20,7 +24,7 @@ class Release(Plugin):
|
||||
def __init__(self):
|
||||
addEvent('release.add', self.add)
|
||||
|
||||
addApiView('release.download', self.download, docs = {
|
||||
addApiView('release.manual_download', self.manualDownload, docs = {
|
||||
'desc': 'Send a release manually to the downloaders',
|
||||
'params': {
|
||||
'id': {'type': 'id', 'desc': 'ID of the release object in release-table'}
|
||||
@@ -45,6 +49,9 @@ class Release(Plugin):
|
||||
}
|
||||
})
|
||||
|
||||
addEvent('release.download', self.download)
|
||||
addEvent('release.try_download_result', self.tryDownloadResult)
|
||||
addEvent('release.create_from_search', self.createFromSearch)
|
||||
addEvent('release.for_movie', self.forMovie)
|
||||
addEvent('release.delete', self.delete)
|
||||
addEvent('release.clean', self.clean)
|
||||
@@ -138,7 +145,6 @@ class Release(Plugin):
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def saveFile(self, filepath, type = 'unknown', include_media_info = False):
|
||||
|
||||
properties = {}
|
||||
@@ -199,7 +205,7 @@ class Release(Plugin):
|
||||
'success': True
|
||||
}
|
||||
|
||||
def download(self, id = None, **kwargs):
|
||||
def manualDownload(self, id = None, **kwargs):
|
||||
|
||||
db = get_session()
|
||||
|
||||
@@ -211,7 +217,7 @@ class Release(Plugin):
|
||||
for info in rel.info:
|
||||
item[info.identifier] = info.value
|
||||
|
||||
fireEvent('notify.frontend', type = 'release.download', data = True, message = 'Snatching "%s"' % item['name'])
|
||||
fireEvent('notify.frontend', type = 'release.manual_download', data = True, message = 'Snatching "%s"' % item['name'])
|
||||
|
||||
# Get matching provider
|
||||
provider = fireEvent('provider.belongs_to', item['url'], provider = item.get('provider'), single = True)
|
||||
@@ -223,7 +229,7 @@ class Release(Plugin):
|
||||
if item.get('protocol') != 'torrent_magnet':
|
||||
item['download'] = provider.loginDownload if provider.urls.get('login') else provider.download
|
||||
|
||||
success = fireEvent('searcher.download', data = item, movie = rel.movie.to_dict({
|
||||
success = fireEvent('release.download', data = item, media = rel.movie.to_dict({
|
||||
'profile': {'types': {'quality': {}}},
|
||||
'releases': {'status': {}, 'quality': {}},
|
||||
'library': {'titles': {}, 'files':{}},
|
||||
@@ -234,7 +240,7 @@ class Release(Plugin):
|
||||
db.expunge_all()
|
||||
rel = db.query(Relea).filter_by(id = id).first() # Get release again @RuudBurger why do we need to get it again??
|
||||
|
||||
fireEvent('notify.frontend', type = 'release.download', data = True, message = 'Successfully snatched "%s"' % item['name'])
|
||||
fireEvent('notify.frontend', type = 'release.manual_download', data = True, message = 'Successfully snatched "%s"' % item['name'])
|
||||
return {
|
||||
'success': success
|
||||
}
|
||||
@@ -245,6 +251,152 @@ class Release(Plugin):
|
||||
'success': False
|
||||
}
|
||||
|
||||
def download(self, data, media, manual = False):
|
||||
|
||||
if not data.get('protocol'):
|
||||
data['protocol'] = data['type']
|
||||
data['type'] = 'movie'
|
||||
|
||||
# Test to see if any downloaders are enabled for this type
|
||||
downloader_enabled = fireEvent('download.enabled', manual, data, single = True)
|
||||
|
||||
if downloader_enabled:
|
||||
snatched_status, done_status, active_status = fireEvent('status.get', ['snatched', 'done', 'active'], single = True)
|
||||
|
||||
# Download release to temp
|
||||
filedata = None
|
||||
if data.get('download') and (ismethod(data.get('download')) or isfunction(data.get('download'))):
|
||||
filedata = data.get('download')(url = data.get('url'), nzb_id = data.get('id'))
|
||||
if filedata == 'try_next':
|
||||
return filedata
|
||||
|
||||
download_result = fireEvent('download', data = data, movie = media, manual = manual, filedata = filedata, single = True)
|
||||
log.debug('Downloader result: %s', download_result)
|
||||
|
||||
if download_result:
|
||||
try:
|
||||
# Mark release as snatched
|
||||
db = get_session()
|
||||
rls = db.query(Relea).filter_by(identifier = md5(data['url'])).first()
|
||||
if rls:
|
||||
renamer_enabled = Env.setting('enabled', 'renamer')
|
||||
|
||||
# Save download-id info if returned
|
||||
if isinstance(download_result, dict):
|
||||
for key in download_result:
|
||||
rls_info = ReleaseInfo(
|
||||
identifier = 'download_%s' % key,
|
||||
value = toUnicode(download_result.get(key))
|
||||
)
|
||||
rls.info.append(rls_info)
|
||||
db.commit()
|
||||
|
||||
log_movie = '%s (%s) in %s' % (getTitle(media['library']), media['library']['year'], rls.quality.label)
|
||||
snatch_message = 'Snatched "%s": %s' % (data.get('name'), log_movie)
|
||||
log.info(snatch_message)
|
||||
fireEvent('%s.snatched' % data['type'], message = snatch_message, data = rls.to_dict())
|
||||
|
||||
# If renamer isn't used, mark media done
|
||||
if not renamer_enabled:
|
||||
try:
|
||||
if media['status_id'] == active_status.get('id'):
|
||||
for profile_type in media['profile']['types']:
|
||||
if profile_type['quality_id'] == rls.quality.id and profile_type['finish']:
|
||||
log.info('Renamer disabled, marking media as finished: %s', log_movie)
|
||||
|
||||
# Mark release done
|
||||
fireEvent('release.update_status', rls.id, status = done_status, single = True)
|
||||
|
||||
# Mark media done
|
||||
mdia = db.query(Media).filter_by(id = media['id']).first()
|
||||
mdia.status_id = done_status.get('id')
|
||||
mdia.last_edit = int(time.time())
|
||||
db.commit()
|
||||
except:
|
||||
log.error('Failed marking media finished, renamer disabled: %s', traceback.format_exc())
|
||||
else:
|
||||
fireEvent('release.update_status', rls.id, status = snatched_status, single = True)
|
||||
|
||||
except:
|
||||
log.error('Failed marking media finished: %s', traceback.format_exc())
|
||||
|
||||
return True
|
||||
|
||||
log.info('Tried to download, but none of the "%s" downloaders are enabled or gave an error', (data.get('protocol')))
|
||||
|
||||
return False
|
||||
|
||||
def tryDownloadResult(self, results, media, quality_type, manual = False):
|
||||
ignored_status, failed_status = fireEvent('status.get', ['ignored', 'failed'], single = True)
|
||||
|
||||
for rel in results:
|
||||
if not quality_type.get('finish', False) and quality_type.get('wait_for', 0) > 0 and rel.get('age') <= quality_type.get('wait_for', 0):
|
||||
log.info('Ignored, waiting %s days: %s', (quality_type.get('wait_for'), rel['name']))
|
||||
continue
|
||||
|
||||
if rel['status_id'] in [ignored_status.get('id'), failed_status.get('id')]:
|
||||
log.info('Ignored: %s', rel['name'])
|
||||
continue
|
||||
|
||||
if rel['score'] <= 0:
|
||||
log.info('Ignored, score to low: %s', rel['name'])
|
||||
continue
|
||||
|
||||
downloaded = fireEvent('release.download', data = rel, media = media, manual = manual, single = True)
|
||||
if downloaded is True:
|
||||
return True
|
||||
elif downloaded != 'try_next':
|
||||
break
|
||||
|
||||
return False
|
||||
|
||||
def createFromSearch(self, search_results, media, quality_type):
|
||||
|
||||
available_status = fireEvent('status.get', ['available'], single = True)
|
||||
db = get_session()
|
||||
|
||||
found_releases = []
|
||||
|
||||
for rel in search_results:
|
||||
|
||||
rel_identifier = md5(rel['url'])
|
||||
found_releases.append(rel_identifier)
|
||||
|
||||
rls = db.query(Relea).filter_by(identifier = rel_identifier).first()
|
||||
if not rls:
|
||||
rls = Relea(
|
||||
identifier = rel_identifier,
|
||||
movie_id = media.get('id'),
|
||||
#media_id = media.get('id'),
|
||||
quality_id = quality_type.get('quality_id'),
|
||||
status_id = available_status.get('id')
|
||||
)
|
||||
db.add(rls)
|
||||
else:
|
||||
[db.delete(old_info) for old_info in rls.info]
|
||||
rls.last_edit = int(time.time())
|
||||
|
||||
db.commit()
|
||||
|
||||
for info in rel:
|
||||
try:
|
||||
if not isinstance(rel[info], (str, unicode, int, long, float)):
|
||||
continue
|
||||
|
||||
rls_info = ReleaseInfo(
|
||||
identifier = info,
|
||||
value = toUnicode(rel[info])
|
||||
)
|
||||
rls.info.append(rls_info)
|
||||
except InterfaceError:
|
||||
log.debug('Couldn\'t add %s to ReleaseInfo: %s', (info, traceback.format_exc()))
|
||||
|
||||
db.commit()
|
||||
|
||||
rel['status_id'] = rls.status_id
|
||||
|
||||
return found_releases
|
||||
|
||||
def forMovie(self, id = None):
|
||||
|
||||
db = get_session()
|
||||
|
||||
@@ -256,7 +256,10 @@ class YarrProvider(Provider):
|
||||
if identifier in qualities:
|
||||
return ids
|
||||
|
||||
return [self.cat_backup_id]
|
||||
if self.cat_backup_id:
|
||||
return [self.cat_backup_id]
|
||||
|
||||
return []
|
||||
|
||||
|
||||
class ResultList(list):
|
||||
@@ -284,12 +287,23 @@ class ResultList(list):
|
||||
|
||||
new_result = self.fillResult(result)
|
||||
|
||||
is_correct_movie = fireEvent('searcher.correct_release', new_result, self.movie, self.quality,
|
||||
is_correct = fireEvent('searcher.correct_release', new_result, self.movie, self.quality,
|
||||
imdb_results = self.kwargs.get('imdb_results', False), single = True)
|
||||
|
||||
if is_correct_movie and new_result['id'] not in self.result_ids:
|
||||
if is_correct and new_result['id'] not in self.result_ids:
|
||||
is_correct_weight = float(is_correct)
|
||||
|
||||
new_result['score'] += fireEvent('score.calculate', new_result, self.movie, single = True)
|
||||
|
||||
old_score = new_result['score']
|
||||
new_result['score'] = int(old_score * is_correct_weight)
|
||||
|
||||
log.info('Found correct release with weight %.02f, old_score(%d) now scaled to score(%d)', (
|
||||
is_correct_weight,
|
||||
old_score,
|
||||
new_result['score']
|
||||
))
|
||||
|
||||
self.found(new_result)
|
||||
self.result_ids.append(result['id'])
|
||||
|
||||
|
||||
Reference in New Issue
Block a user