From 2393b43ebf69ab1fd8b939191e78f14071a01aed Mon Sep 17 00:00:00 2001 From: Ruud Date: Fri, 18 May 2012 22:23:59 +0200 Subject: [PATCH] Try and catch newznab limits. fixes #292 #264 --- .../core/downloaders/blackhole/main.py | 10 +++---- couchpotato/core/downloaders/nzbget/main.py | 19 ++++-------- couchpotato/core/downloaders/sabnzbd/main.py | 14 ++++----- .../core/downloaders/transmission/main.py | 6 ++-- couchpotato/core/plugins/searcher/main.py | 15 ++++++++-- .../core/providers/nzb/newznab/main.py | 29 +++++++++++++++++++ 6 files changed, 60 insertions(+), 33 deletions(-) diff --git a/couchpotato/core/downloaders/blackhole/main.py b/couchpotato/core/downloaders/blackhole/main.py index a24e71b1..c12c8c6e 100644 --- a/couchpotato/core/downloaders/blackhole/main.py +++ b/couchpotato/core/downloaders/blackhole/main.py @@ -10,7 +10,7 @@ class Blackhole(Downloader): type = ['nzb', 'torrent'] - def download(self, data = {}, movie = {}, manual = False): + def download(self, data = {}, movie = {}, manual = False, filedata = None): if self.isDisabled(manual) or (not self.isCorrectType(data.get('type')) or (not self.conf('use_for') in ['both', data.get('type')])): return @@ -19,10 +19,8 @@ class Blackhole(Downloader): log.error('No directory set for blackhole %s download.' % data.get('type')) else: try: - filedata = data.get('download')(url = data.get('url'), nzb_id = data.get('id')) - - if len(filedata) < 50: - log.error('No nzb available!') + if not filedata or len(filedata) < 50: + log.error('No nzb/torrent available!') return False fullPath = os.path.join(directory, self.createFileName(data, filedata, movie)) @@ -42,6 +40,6 @@ class Blackhole(Downloader): pass except: - log.debug('Failed to download file %s: %s' % (data.get('name'), traceback.format_exc())) + log.info('Failed to download file %s: %s' % (data.get('name'), traceback.format_exc())) return False return False diff --git a/couchpotato/core/downloaders/nzbget/main.py b/couchpotato/core/downloaders/nzbget/main.py index 15243ca4..904a3a4e 100644 --- a/couchpotato/core/downloaders/nzbget/main.py +++ b/couchpotato/core/downloaders/nzbget/main.py @@ -14,11 +14,15 @@ class NZBGet(Downloader): url = 'http://nzbget:%(password)s@%(host)s/xmlrpc' - def download(self, data = {}, movie = {}, manual = False): + def download(self, data = {}, movie = {}, manual = False, filedata = None): if self.isDisabled(manual) or not self.isCorrectType(data.get('type')): return + if not filedata: + log.error('Unable to get NZB file: %s' % traceback.format_exc()) + return False + log.info('Sending "%s" to NZBGet.' % data.get('name')) url = self.url % {'host': self.conf('host'), 'password': self.conf('password')} @@ -40,19 +44,6 @@ class NZBGet(Downloader): log.error('Protocol Error: %s' % e) return False - try: - if isfunction(data.get('download')): - filedata = data.get('download')() - if not filedata: - log.error('Failed download file: %s' % nzb_name) - return False - else: - log.info('Downloading: %s' % data.get('url')) - filedata = self.urlopen(data.get('url')) - except: - log.error('Unable to get NZB file: %s' % traceback.format_exc()) - return False - if rpc.append(nzb_name, self.conf('category'), False, standard_b64encode(filedata.strip())): log.info('NZB sent successfully to NZBGet') return True diff --git a/couchpotato/core/downloaders/sabnzbd/main.py b/couchpotato/core/downloaders/sabnzbd/main.py index 5505438e..d6c5af6c 100644 --- a/couchpotato/core/downloaders/sabnzbd/main.py +++ b/couchpotato/core/downloaders/sabnzbd/main.py @@ -15,7 +15,7 @@ class Sabnzbd(Downloader): type = ['nzb'] - def download(self, data = {}, movie = {}, manual = False): + def download(self, data = {}, movie = {}, manual = False, filedata = None): if self.isDisabled(manual) or not self.isCorrectType(data.get('type')): return @@ -42,15 +42,13 @@ class Sabnzbd(Downloader): 'nzbname': self.createNzbName(data, movie), } - if data.get('download') and (ismethod(data.get('download')) or isfunction(data.get('download'))): - nzb_file = data.get('download')(url = data.get('url'), nzb_id = data.get('id')) - - if not nzb_file or len(nzb_file) < 50: - log.error('No nzb available!') + if filedata: + if len(filedata) < 50: + log.error('No proper nzb available!') return False # If it's a .rar, it adds the .rar extension, otherwise it stays .nzb - nzb_filename = self.createFileName(data, nzb_file, movie) + nzb_filename = self.createFileName(data, filedata, movie) params['mode'] = 'addfile' else: params['name'] = data.get('url') @@ -62,7 +60,7 @@ class Sabnzbd(Downloader): try: if params.get('mode') is 'addfile': - data = self.urlopen(url, params = {"nzbfile": (nzb_filename, nzb_file)}, multipart = True, show_error = False) + data = self.urlopen(url, params = {"nzbfile": (nzb_filename, filedata)}, multipart = True, show_error = False) else: data = self.urlopen(url, show_error = False) except: diff --git a/couchpotato/core/downloaders/transmission/main.py b/couchpotato/core/downloaders/transmission/main.py index 4ed3e583..55e4e305 100644 --- a/couchpotato/core/downloaders/transmission/main.py +++ b/couchpotato/core/downloaders/transmission/main.py @@ -11,7 +11,7 @@ class Transmission(Downloader): type = ['torrent'] - def download(self, data = {}, movie = {}, manual = False): + def download(self, data = {}, movie = {}, manual = False, filedata = None): if self.isDisabled(manual) or not self.isCorrectType(data.get('type')): return @@ -31,8 +31,10 @@ class Transmission(Downloader): } try: + if not filedata: + log.error('Failed sending torrent to transmission, no data') + tc = transmissionrpc.Client(host[0], port = host[1], user = self.conf('username'), password = self.conf('password')) - filedata = data.get('download')(url = data.get('url'), nzb_id = data.get('id')) torrent = tc.add_torrent(b64encode(filedata), **params) # Change settings of added torrents diff --git a/couchpotato/core/plugins/searcher/main.py b/couchpotato/core/plugins/searcher/main.py index e2933462..57e1f37c 100644 --- a/couchpotato/core/plugins/searcher/main.py +++ b/couchpotato/core/plugins/searcher/main.py @@ -6,6 +6,7 @@ from couchpotato.core.logger import CPLog from couchpotato.core.plugins.base import Plugin from couchpotato.core.settings.model import Movie, Release, ReleaseInfo from couchpotato.environment import Env +from inspect import ismethod, isfunction from sqlalchemy.exc import InterfaceError import datetime import re @@ -142,9 +143,9 @@ class Searcher(Plugin): for nzb in sorted_results: downloaded = self.download(data = nzb, movie = movie) - if downloaded: + if downloaded is True: return True - else: + elif downloaded != 'try_next': break else: log.info('Better quality (%s) already available or snatched for %s' % (quality_type['quality']['label'], default_title)) @@ -161,7 +162,15 @@ class Searcher(Plugin): def download(self, data, movie, manual = False): snatched_status = fireEvent('status.get', 'snatched', single = True) - successful = fireEvent('download', data = data, movie = movie, manual = manual, single = True) + + # Download movie to temp + filedata = None + if data.get('download') and (ismethod(data.get('download')) or isfunction(data.get('download'))): + filedata = data.get('download')(url = data.get('url'), nzb_id = data.get('id')) + if filedata is 'try_next': + return filedata + + successful = fireEvent('download', data = data, movie = movie, manual = manual, single = True, filedata = filedata) if successful: diff --git a/couchpotato/core/providers/nzb/newznab/main.py b/couchpotato/core/providers/nzb/newznab/main.py index 0d648a95..8b959b54 100644 --- a/couchpotato/core/providers/nzb/newznab/main.py +++ b/couchpotato/core/providers/nzb/newznab/main.py @@ -6,7 +6,10 @@ from couchpotato.core.logger import CPLog from couchpotato.core.providers.nzb.base import NZBProvider from couchpotato.environment import Env from dateutil.parser import parse +from urllib2 import HTTPError +from urlparse import urlparse import time +import traceback import xml.etree.ElementTree as XMLTree log = CPLog(__name__) @@ -20,6 +23,8 @@ class Newznab(NZBProvider, RSS): 'search': 'movie', } + limits_reached = {} + cat_ids = [ ([2010], ['dvdr']), ([2030], ['cam', 'ts', 'dvdrip', 'tc', 'r5', 'scr']), @@ -194,3 +199,27 @@ class Newznab(NZBProvider, RSS): def getApiExt(self, host): return '&apikey=%s' % host['api_key'] + + def download(self, url = '', nzb_id = ''): + host = urlparse(url).hostname + + if self.limits_reached.get(host): + # Try again in 3 hours + if self.limits_reached[host] > time.time() - 10800: + return 'try_next' + + try: + data = self.urlopen(url, show_error = False) + self.limits_reached[host] = False + return data + except HTTPError, e: + if e.code == 503: + response = e.read().lower() + if 'maximum api' in response or 'download limit' in response: + if not self.limits_reached.get(host): + log.error('Limit reached for newznab provider: %s' % host) + self.limits_reached[host] = time.time() + return 'try_next' + + log.error('Failed download from %s' % (host, traceback.format_exc())) + raise