From d84897ff335db2c26739abb90d7d304cc584867f Mon Sep 17 00:00:00 2001 From: Dustin Brewer Date: Sun, 21 Dec 2014 13:25:06 -0800 Subject: [PATCH 01/60] Initial support for Plex Media Server w/Plex Home --- .../core/notifications/plex/__init__.py | 20 +++++++++ couchpotato/core/notifications/plex/server.py | 43 +++++++++++++++++-- 2 files changed, 59 insertions(+), 4 deletions(-) diff --git a/couchpotato/core/notifications/plex/__init__.py b/couchpotato/core/notifications/plex/__init__.py index 4a64ec5b..957369b7 100755 --- a/couchpotato/core/notifications/plex/__init__.py +++ b/couchpotato/core/notifications/plex/__init__.py @@ -23,6 +23,26 @@ config = [{ 'default': 'localhost', 'description': 'Hostname/IP, default localhost' }, + { + 'name': 'username', + 'label': 'Username', + 'default': '', + 'description': 'Required for myPlex' + }, + { + 'name': 'password', + 'label': 'Password', + 'default': '', + 'type': 'password', + 'description': 'Required for myPlex' + }, + { + 'name': 'auth_token', + 'label': 'Auth Token', + 'default': '', + 'advanced': True, + 'description': 'Required for myPlex' + }, { 'name': 'clients', 'default': '', diff --git a/couchpotato/core/notifications/plex/server.py b/couchpotato/core/notifications/plex/server.py index cd11f49b..4b8ea05f 100644 --- a/couchpotato/core/notifications/plex/server.py +++ b/couchpotato/core/notifications/plex/server.py @@ -35,11 +35,46 @@ class PlexServer(object): if path.startswith('/'): path = path[1:] - data = self.plex.urlopen('%s/%s' % ( - self.createHost(self.plex.conf('media_server'), port = 32400), - path - )) + #Maintain support for older Plex installations without myPlex + if not self.plex.conf('auth_token') and not self.plex.conf('username') and not self.plex.conf('password'): + data = self.plex.urlopen('%s/%s' % ( + self.createHost(self.plex.conf('media_server'), port = 32400), + path + )) + else: + #Fetch X-Plex-Token if it doesn't exist but a username/password do + if not self.plex.conf('auth_token') and (self.plex.conf('username') and self.plex.conf('password')): + import urllib2, base64 + log.info("Fetching a new X-Plex-Token from plex.tv") + username = self.plex.conf('username') + password = self.plex.conf('password') + req = urllib2.Request("https://plex.tv/users/sign_in.xml", data="") + authheader = "Basic %s" % base64.encodestring('%s:%s' % (username, password))[:-1] + req.add_header("Authorization", authheader) + req.add_header("X-Plex-Product", "Couchpotato Notifier") + req.add_header("X-Plex-Client-Identifier", "b3a6b24dcab2224bdb101fc6aa08ea5e2f3147d6") + req.add_header("X-Plex-Version", "1.0") + + try: + response = urllib2.urlopen(req) + except urllib2.URLError, e: + log.info("Error fetching token from plex.tv") + + try: + auth_tree = etree.parse(response) + token = auth_tree.findall(".//authentication-token")[0].text + self.plex.conf('auth_token', token) + except (ValueError, IndexError) as e: + log.info("Error parsing plex.tv response: " + ex(e)) + + #Add X-Plex-Token header for myPlex support workaround + data = self.plex.urlopen('%s/%s?X-Plex-Token=%s' % ( + self.createHost(self.plex.conf('media_server'), port = 32400), + path, + self.plex.conf('auth_token') + )) + if data_type == 'xml': return etree.fromstring(data) else: From 9318e1934791f608fbf5c917b44838700695e8e9 Mon Sep 17 00:00:00 2001 From: jonnyboy Date: Wed, 31 Dec 2014 08:21:58 -0500 Subject: [PATCH 02/60] New torrent search provider hdaccess.net --- .../media/_base/providers/torrent/hdaccess.py | 150 ++++++++++++++++++ .../media/movie/providers/torrent/hdaccess.py | 11 ++ 2 files changed, 161 insertions(+) create mode 100644 couchpotato/core/media/_base/providers/torrent/hdaccess.py create mode 100644 couchpotato/core/media/movie/providers/torrent/hdaccess.py diff --git a/couchpotato/core/media/_base/providers/torrent/hdaccess.py b/couchpotato/core/media/_base/providers/torrent/hdaccess.py new file mode 100644 index 00000000..811c599c --- /dev/null +++ b/couchpotato/core/media/_base/providers/torrent/hdaccess.py @@ -0,0 +1,150 @@ +import re +import json +import traceback + +from couchpotato.core.helpers.variable import tryInt, getIdentifier +from couchpotato.core.logger import CPLog +from couchpotato.core.media._base.providers.torrent.base import TorrentProvider + + +log = CPLog(__name__) + + +class Base(TorrentProvider): + + urls = { + 'test': 'https://hdaccess.net/', + 'detail': 'https://hdaccess.net/details.php?id=%s', + 'search': 'https://hdaccess.net/searchapi.php?apikey=%s&username=%s&imdbid=%s&internal=%s', + 'download': 'https://hdaccess.net/grab.php?torrent=%s&apikey=%s', + } + + http_time_between_calls = 1 # Seconds + + def _search(self, movie, quality, results): + data = self.getJsonData(self.urls['search'] % (self.conf('apikey'), self.conf('username'), getIdentifier(movie), self.conf('internal_only'))) + + if data: + try: + #for result in data[]: + for key, result in data.iteritems(): + if tryInt(result['total_results']) == 0: + return + torrentscore = self.conf('extra_score') + releasegroup = result['releasegroup'] + resolution = result['resolution'] + encoding = result['encoding'] + freeleech = tryInt(result['freeleech']) + seeders = tryInt(result['seeders']) + torrent_desc = '/ %s / %s / %s / %s seeders' % (releasegroup, resolution, encoding, seeders) + + if freeleech > 0 and self.conf('prefer_internal'): + torrent_desc += '/ Internal' + torrentscore += 200 + + if resolution == '1080p': + torrentscore += 100 + if encoding == 'x264' and self.conf('favor') in ['encode', 'all']: + torrentscore += 100 + elif encoding == 'Encode' and self.conf('favor') in ['encode', 'all']: + torrentscore += 100 + elif encoding == 'Remux' and self.conf('favor') in ['remux', 'all']: + torrentscore += 200 + elif encoding == 'Bluray' and self.conf('favor') in ['bluray', 'all']: + torrentscore += 300 + + if seeders == 0: + torrentscore = 0 + + name = result['release_name'] + year = tryInt(result['year']) + + results.append({ + 'id': tryInt(result['torrentid']), + 'name': re.sub('[^A-Za-z0-9\-_ \(\).]+', '', '%s (%s) %s' % (name, year, torrent_desc)), + 'url': self.urls['download'] % (result['torrentid'], self.conf('apikey')), + 'detail_url': self.urls['detail'] % result['torrentid'], + 'size': tryInt(result['size']), + 'seeders': tryInt(result['seeders']), + 'leechers': tryInt(result['leechers']), + 'age': tryInt(result['age']), + 'score': torrentscore + }) + except: + log.error('Failed getting results from %s: %s', (self.getName(), traceback.format_exc())) +config = [{ + 'name': 'hdaccess', + 'groups': [ + { + 'tab': 'searcher', + 'list': 'torrent_providers', + 'name': 'HDAccess', + 'wizard': True, + 'description': 'HDAccess', + 'icon': 'iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABmJLR0QA/wD/AP+gvaeTAAADuUlEQVQ4yz3T209bdQAH8O/vnNNzWno5FIpAKZdSLi23gWMDtumWuSXOyzJj9M1kyIOPS1xiYuKe9GUPezZZnGIiMTqTxS1bdIuYkG2MWKBAKYVszOgKFkrbA+259HfO+fli/PwPHzI+Pg5CCEAI2VcUlEsl1tHdU7P5bGOkWChEaaUCwvHpmkD93POn6bwgCMQGAMYYYwyCruuQnE7SPzjIstvb8l+bm5fXkokJSmlQEkUQAIpSRH5vd0tyum7I/sA1Z5VH2ctmiGWZjHw4McE1NAZtQ9fD25kXt1VN7es7dNjuGRjiJFeVpWo6slsZPhF/Ys/PPeIs2056ff7zIOS5rpU5/viJEwwEnu3Mi18dojjw0aWP6amz57h9RSE/35zinq2nuGjvIQwOj7K2SKeZWkk0auXSSZ+/ZopSy+CbW1pQKpWu6Jr2/qVPPqWRjm6HWi6Tm999g3RyGbndLCqGgVBrO3F7fHykK0YX47NNtGLYlBq/c+H2iD+3k704dHQUDcFmQVXLyP6zhfTqCl45fQYjx17FemoJunoAk1bQFGoVhkdPwNC0ix2dMT+3llodM02rKdo7gN3dHAEhuH/vNgDg3Pl3cPaNt2GZJpYX5lBbFwClBukfGobL5WrayW6NccVCISY4HIQxYts2Q3J5CXOPHuLlo6NoCoXQ2hbG0JFRpJYWcVDIQ5ZlyL5qW5b9hNlWjKsYBgzDgKppMCoGHty7A0orOHbyNNweL+obGnDm9TdhWSYS8Vn4a2shOZ0QJRGSKIHjeGGtWNhjqqpyG+k04k8eozPai9ZwByavf4kfpyZxZGwMfYOHsbwQx34hB5dL4syKweRq/xpXHwzNapqWSSYWMDszzYqFPEaOn4KiKJiZfoCZ6d8Am+GtC++iXCpjaf4P9vefT8HzfKarp3eWRKMxCILwuWXSz977YIK2RTodDoGH1+OG1+tDlbsKkuiAJEngeWBjNUUnv7rucIiOLyzTvMKJTgnVtbVXLctK3L31g+NAUajL5bEptaDpOnTdgGkzVHl9drms0ju3fnJIkphoaQtfbQiFwAcCAY5wnCE5Xff3i8XX4o9nGksH+8zl9hAGZlWMCivkc9z0L3fZ999+LTCGZKi55YJTFHfye3sc6e/vB88LpK6+iWlqSS4WcpcNXZtwOp3B6mo/REmCSSkEgd+qq3vpRkt75Fp9Y1BZWZwnhq4zEovF/u/MATAti4U7umvyu9kR27aikihC9vvTnV2xufVUMu/2uIksy/9tZvgX49fLmAMx3bsAAAAASUVORK5CYII=', + 'options': [ + { + 'name': 'enabled', + 'type': 'enabler', + 'default': False, + }, + { + 'name': 'username', + 'default': '', + 'description': 'Enter your site username.', + }, + { + 'name': 'apikey', + 'default': '', + 'label': 'API Key', + 'description': 'Enter your site api key. This can be find in \'Edit My Profile\'->Security', + }, + { + 'name': 'seed_ratio', + 'label': 'Seed ratio', + 'type': 'float', + 'default': 0, + 'description': 'Will not be (re)moved until this seed ratio is met. HDAccess minimum is 1:1.', + }, + { + 'name': 'seed_time', + 'label': 'Seed time', + 'type': 'int', + 'default': 0, + 'description': 'Will not be (re)moved until this seed time (in hours) is met. HDAccess minimum is 48 hours.', + }, + { + 'name': 'prefer_internal', + 'advanced': True, + 'type': 'bool', + 'default': 1, + 'description': 'Favors internal releases over non-internal releases.', + }, + { + 'name': 'favor', + 'advanced': True, + 'default': 'all', + 'type': 'dropdown', + 'values': [('Blurays & Encodes & Remuxes', 'all'), ('Blurays', 'bluray'), ('Encodes', 'encode'), ('Remuxes', 'remux'), ('None', 'none')], + 'description': 'Give extra scoring to blurays(+300), remuxes(+200) or encodes(+100).', + }, + { + 'name': 'internal_only', + 'advanced': True, + 'label': 'Internal Only', + 'type': 'bool', + 'default': False, + 'description': 'Only download releases marked as HDAccess internal', + }, + { + 'name': 'extra_score', + 'advanced': True, + 'label': 'Extra Score', + 'type': 'int', + 'default': 0, + 'description': 'Starting score for each release found via this provider.', + } + ], + }, + ], +}] diff --git a/couchpotato/core/media/movie/providers/torrent/hdaccess.py b/couchpotato/core/media/movie/providers/torrent/hdaccess.py new file mode 100644 index 00000000..fae2cf54 --- /dev/null +++ b/couchpotato/core/media/movie/providers/torrent/hdaccess.py @@ -0,0 +1,11 @@ +from couchpotato.core.logger import CPLog +from couchpotato.core.media._base.providers.torrent.hdaccess import Base +from couchpotato.core.media.movie.providers.base import MovieProvider + +log = CPLog(__name__) + +autoload = 'HDAccess' + + +class HDAccess(MovieProvider, Base): + pass From 17e37996c4b31974050d560685bf0c52d41dabe6 Mon Sep 17 00:00:00 2001 From: Ruud Burger Date: Fri, 2 Jan 2015 18:18:08 +0100 Subject: [PATCH 03/60] Add remux category for TorrentShack close #4427 --- .../core/media/movie/providers/torrent/torrentshack.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/couchpotato/core/media/movie/providers/torrent/torrentshack.py b/couchpotato/core/media/movie/providers/torrent/torrentshack.py index 01eb6d6a..f9127315 100644 --- a/couchpotato/core/media/movie/providers/torrent/torrentshack.py +++ b/couchpotato/core/media/movie/providers/torrent/torrentshack.py @@ -22,8 +22,8 @@ class TorrentShack(MovieProvider, Base): # Movies-SD Pack - 983 (not included) cat_ids = [ - ([970], ['bd50']), - ([300], ['720p', '1080p']), + ([970, 320], ['bd50']), + ([300, 320], ['720p', '1080p']), ([350], ['dvdr']), ([400], ['brrip', 'dvdrip']), ] From 038b4c63eee7b12a0ac6de62bc8305122a4ed3af Mon Sep 17 00:00:00 2001 From: Andrew Dumaresq Date: Sun, 4 Jan 2015 17:09:36 -0500 Subject: [PATCH 04/60] Updated to follow putio API changes --- libs/pio/api.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/libs/pio/api.py b/libs/pio/api.py index 0f2a2c66..ecfc1776 100644 --- a/libs/pio/api.py +++ b/libs/pio/api.py @@ -154,13 +154,14 @@ class _File(_BaseResource): return [cls(f) for f in files] @classmethod - def upload(cls, path, name=None): + def upload(cls, path, name=None, parent_id=0): with open(path) as f: if name: files = {'file': (name, f)} else: files = {'file': f} - d = cls.client.request('/files/upload', method='POST', files=files) + d = cls.client.request('/files/upload', method='POST', + data={'parent_id': parent_id}, files=files) f = d['file'] return cls(f) @@ -239,7 +240,7 @@ class _Transfer(_BaseResource): @classmethod def add_url(cls, url, parent_id=0, extract=False, callback_url=None): d = cls.client.request('/transfers/add', method='POST', data=dict( - url=url, parent_id=parent_id, extract=extract, + url=url, save_parent_id=parent_id, extract=extract, callback_url=callback_url)) t = d['transfer'] return cls(t) @@ -249,7 +250,7 @@ class _Transfer(_BaseResource): with open(path) as f: files = {'file': f} d = cls.client.request('/files/upload', method='POST', files=files, - data=dict(parent_id=parent_id, + data=dict(save_parent_id=parent_id, extract=extract, callback_url=callback_url)) t = d['transfer'] From d012dc5c85aa88538990da01b41f1a664366e3d7 Mon Sep 17 00:00:00 2001 From: Andrew Dumaresq Date: Sun, 4 Jan 2015 17:10:16 -0500 Subject: [PATCH 05/60] Added new folder option --- couchpotato/core/downloaders/putio/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/couchpotato/core/downloaders/putio/__init__.py b/couchpotato/core/downloaders/putio/__init__.py index 114ad6d8..60ccad58 100644 --- a/couchpotato/core/downloaders/putio/__init__.py +++ b/couchpotato/core/downloaders/putio/__init__.py @@ -28,6 +28,11 @@ config = [{ 'description': 'This is the OAUTH_TOKEN from your putio API', 'advanced': True, }, + { + 'name': 'folder', + 'description': ('The folder on putio where you want the upload to go','Must be a folder in the root directory'), + 'default': 0, + }, { 'name': 'callback_host', 'description': 'External reachable url to CP so put.io can do it\'s thing', From 2c72cd7d9f1a38fcf77695655ce1875de40cbd32 Mon Sep 17 00:00:00 2001 From: Andrew Dumaresq Date: Sun, 4 Jan 2015 17:10:40 -0500 Subject: [PATCH 06/60] Added new folder option and fixed but in callback url --- couchpotato/core/downloaders/putio/main.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/couchpotato/core/downloaders/putio/main.py b/couchpotato/core/downloaders/putio/main.py index 76ac2033..ce58ff7c 100644 --- a/couchpotato/core/downloaders/putio/main.py +++ b/couchpotato/core/downloaders/putio/main.py @@ -28,20 +28,32 @@ class PutIO(DownloaderBase): return super(PutIO, self).__init__() + def convertFolder(self, client, folder): + if folder == 0: + return 0 + else: + files = client.File.list() + for f in files: + if f.name == folder and f.content_type == "application/x-directory": + return f.id + #If we get through the whole list and don't get a match we will use the root + return 0 + def download(self, data = None, media = None, filedata = None): if not media: media = {} if not data: data = {} log.info('Sending "%s" to put.io', data.get('name')) url = data.get('url') - client = pio.Client(self.conf('oauth_token')) + putioFolder = self.convertFolder(client, self.conf('folder')) + log.debug('putioFolder ID is %s', putioFolder) # It might be possible to call getFromPutio from the renamer if we can then we don't need to do this. # Note callback_host is NOT our address, it's the internet host that putio can call too callbackurl = None if self.conf('download'): - callbackurl = 'http://' + self.conf('callback_host') + '/' + '%sdownloader.putio.getfrom/' %Env.get('api_base'.strip('/')) - resp = client.Transfer.add_url(url, callback_url = callbackurl) + callbackurl = 'http://' + self.conf('callback_host') + '%sdownloader.putio.getfrom/' %Env.get('api_base'.strip('/')) + resp = client.Transfer.add_url(url, callback_url = callbackurl, parent_id = putioFolder) log.debug('resp is %s', resp.id); return self.downloadReturnId(resp.id) @@ -124,7 +136,9 @@ class PutIO(DownloaderBase): client = pio.Client(self.conf('oauth_token')) log.debug('About to get file List') - files = client.File.list() + putioFolder = self.convertFolder(client, self.conf('folder')) + log.debug('PutioFolderID is %s', putioFolder) + files = client.File.list(parent_id=putioFolder) downloaddir = self.conf('download_dir') for f in files: From ac6f295c936e45cd61ba304bccfc3783c24f44d0 Mon Sep 17 00:00:00 2001 From: grasshide Date: Mon, 5 Jan 2015 15:00:40 +0100 Subject: [PATCH 07/60] New algogithm to use some kind of crowd logic on newznab powered providers. --- .../movie/providers/automation/crowdai.py | 94 +++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 couchpotato/core/media/movie/providers/automation/crowdai.py diff --git a/couchpotato/core/media/movie/providers/automation/crowdai.py b/couchpotato/core/media/movie/providers/automation/crowdai.py new file mode 100644 index 00000000..0d33f11b --- /dev/null +++ b/couchpotato/core/media/movie/providers/automation/crowdai.py @@ -0,0 +1,94 @@ +from xml.etree.ElementTree import QName +import datetime +import re + +from couchpotato.core.helpers.rss import RSS +from couchpotato.core.helpers.variable import tryInt, splitString +from couchpotato.core.logger import CPLog +from couchpotato.core.media.movie.providers.automation.base import Automation + + +log = CPLog(__name__) + +autoload = 'CrowdAI' + + +class CrowdAI(Automation, RSS): + + interval = 1800 + + def getIMDBids(self): + + movies = [] + + newznab_namespace = 'http://www.newznab.com/DTD/2010/feeds/attributes/' + urls = dict(zip(splitString(self.conf('automation_urls')), [tryInt(x) for x in splitString(self.conf('automation_urls_use'))])) + + for url in urls: + + if not urls[url]: + continue + + rss_movies = self.getRSSData(url) + + for movie in rss_movies: + + title = "" + description = self.getTextElement(movie, "description") + grabs = 0 + + for item in movie: + if item.attrib.get('name') == 'grabs': + grabs = item.attrib.get('value') + break + + + if int(grabs) > tryInt(self.conf('number_grabs')): + title = re.match( r'.*Title: .a href.*/">(.*) \(\d{4}\).*', description).group(1) + log.info2('%s grabs for movie: %s, enqueue...', (grabs, title)) + year = re.match( r'.*Year: (\d{4}).*', description).group(1) + imdb = self.search(title, year) + + if imdb and self.isMinimalMovie(imdb): + movies.append(imdb['imdb']) + + return movies + + +config = [{ + 'name': 'crowdai', + 'groups': [ + { + 'tab': 'automation', + 'list': 'automation_providers', + 'name': 'crowdai_automation', + 'label': 'CrowdAI', + 'description': 'Imports from any newznab powered NZB providers RSS feed depending on the number of grabs per movie. Go to your newznab site and find the RSS section. Then copy the copy paste the link under "Movies > x264 feed" here.', + 'options': [ + { + 'name': 'automation_enabled', + 'default': False, + 'type': 'enabler', + }, + { + 'name': 'automation_urls_use', + 'label': 'Use', + 'default': '1', + }, + { + 'name': 'automation_urls', + 'label': 'url', + 'type': 'combined', + 'combine': ['automation_urls_use', 'automation_urls'], + 'default': 'http://YOUR_PROVIDER/rss?t=THE_MOVIE_CATEGORY&i=YOUR_USER_ID&r=YOUR_API_KEY&res=2&rls=2&num=100', + }, + { + 'name': 'number_grabs', + 'default': '500', + 'label': 'Grab threshold', + 'description': 'Number of grabs required', + }, + ], + }, + ], +}] From a3af784c18024667fc93ac205b0d8713a6c8df60 Mon Sep 17 00:00:00 2001 From: Steven Lu Date: Tue, 6 Jan 2015 18:33:06 -0500 Subject: [PATCH 08/60] Adding the ability to receive notifications through Webhooks --- couchpotato/core/notifications/webhook.py | 66 +++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 couchpotato/core/notifications/webhook.py diff --git a/couchpotato/core/notifications/webhook.py b/couchpotato/core/notifications/webhook.py new file mode 100644 index 00000000..d970bb53 --- /dev/null +++ b/couchpotato/core/notifications/webhook.py @@ -0,0 +1,66 @@ +from couchpotato.core.helpers.encoding import toUnicode +from couchpotato.core.helpers.variable import getIdentifier +from couchpotato.core.logger import CPLog +from couchpotato.core.notifications.base import Notification + +log = CPLog(__name__) + +autoload = 'Webhook' + +class Webhook(Notification): + + + def notify(self, message = '', data = None, listener = None): + if not data: data = {} + + post_data = { + 'message': toUnicode(message) + } + + if getIdentifier(data): + post_data.update({ + 'imdb_id': getIdentifier(data) + }) + + headers = { + 'Content-type': 'application/x-www-form-urlencoded' + } + + try: + self.urlopen(self.conf('url'), headers = headers, data = post_data, show_error = False) + return True + except: + log.error('Webhook notification failed: %s', traceback.format_exc()) + + return False + + +config = [{ + 'name': 'webhook', + 'groups': [ + { + 'tab': 'notifications', + 'list': 'notification_providers', + 'name': 'webhook', + 'label': 'Webhook', + 'options': [ + { + 'name': 'enabled', + 'default': 0, + 'type': 'enabler', + }, + { + 'name': 'url', + 'description': 'The URL to send notification data to when ' + }, + { + 'name': 'on_snatch', + 'default': 0, + 'type': 'bool', + 'advanced': True, + 'description': 'Also send message when movie is snatched.', + } + ] + } + ] +}] From 4c68566c7767e30c920a8a617410c9783eda4855 Mon Sep 17 00:00:00 2001 From: Ruud Date: Thu, 8 Jan 2015 14:59:53 +0100 Subject: [PATCH 09/60] Use new OMGWTFNZB api fix #4471 --- .../media/_base/providers/nzb/omgwtfnzbs.py | 26 +++++++------------ 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/couchpotato/core/media/_base/providers/nzb/omgwtfnzbs.py b/couchpotato/core/media/_base/providers/nzb/omgwtfnzbs.py index bac0614d..21799787 100644 --- a/couchpotato/core/media/_base/providers/nzb/omgwtfnzbs.py +++ b/couchpotato/core/media/_base/providers/nzb/omgwtfnzbs.py @@ -1,13 +1,9 @@ -from urlparse import urlparse, parse_qs -import time - from couchpotato.core.event import fireEvent from couchpotato.core.helpers.encoding import toUnicode, tryUrlencode from couchpotato.core.helpers.rss import RSS from couchpotato.core.helpers.variable import tryInt from couchpotato.core.logger import CPLog from couchpotato.core.media._base.providers.nzb.base import NZBProvider -from dateutil.parser import parse log = CPLog(__name__) @@ -16,8 +12,7 @@ log = CPLog(__name__) class Base(NZBProvider, RSS): urls = { - 'search': 'https://rss.omgwtfnzbs.org/rss-search.php?%s', - 'detail_url': 'https://omgwtfnzbs.org/details.php?id=%s', + 'search': 'https://api.omgwtfnzbs.org/json/?%s', } http_time_between_calls = 1 # Seconds @@ -47,21 +42,18 @@ class Base(NZBProvider, RSS): 'api': self.conf('api_key', default = ''), }) - nzbs = self.getRSSData(self.urls['search'] % params) + nzbs = self.getJsonData(self.urls['search'] % params) for nzb in nzbs: - enclosure = self.getElement(nzb, 'enclosure').attrib - nzb_id = parse_qs(urlparse(self.getTextElement(nzb, 'link')).query).get('id')[0] - results.append({ - 'id': nzb_id, - 'name': toUnicode(self.getTextElement(nzb, 'title')), - 'age': self.calculateAge(int(time.mktime(parse(self.getTextElement(nzb, 'pubDate')).timetuple()))), - 'size': tryInt(enclosure['length']) / 1024 / 1024, - 'url': enclosure['url'], - 'detail_url': self.urls['detail_url'] % nzb_id, - 'description': self.getTextElement(nzb, 'description') + 'id': nzb.get('nzbid'), + 'name': toUnicode(nzb.get('release')), + 'age': self.calculateAge(tryInt(nzb.get('usenetage'))), + 'size': tryInt(nzb.get('sizebytes')) / 1024 / 1024, + 'url': nzb.get('getnzb'), + 'detail_url': nzb.get('details'), + 'description': nzb.get('weblink') }) From 2c080fec3d3a2fdcf63de5b641ad92dd088ae1e3 Mon Sep 17 00:00:00 2001 From: Ruud Date: Thu, 8 Jan 2015 16:56:38 +0100 Subject: [PATCH 10/60] TorrentBytes nbsp issue fix #4026 --- couchpotato/core/media/_base/providers/torrent/torrentbytes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/media/_base/providers/torrent/torrentbytes.py b/couchpotato/core/media/_base/providers/torrent/torrentbytes.py index 8e2becb2..fadd2ea0 100644 --- a/couchpotato/core/media/_base/providers/torrent/torrentbytes.py +++ b/couchpotato/core/media/_base/providers/torrent/torrentbytes.py @@ -56,7 +56,7 @@ class Base(TorrentProvider): full_id = link['href'].replace('details.php?id=', '') torrent_id = full_id[:6] - name = toUnicode(link.contents[0]) + name = toUnicode(link.contents[0].encode('ISO-8859-1')).strip() results.append({ 'id': torrent_id, From e7b089edf596294e8a3bf8b14981517f8171e038 Mon Sep 17 00:00:00 2001 From: Ruud Date: Fri, 9 Jan 2015 20:13:17 +0100 Subject: [PATCH 11/60] Give better XML issues --- couchpotato/core/media/_base/providers/base.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/couchpotato/core/media/_base/providers/base.py b/couchpotato/core/media/_base/providers/base.py index 587545c8..1062a1a8 100644 --- a/couchpotato/core/media/_base/providers/base.py +++ b/couchpotato/core/media/_base/providers/base.py @@ -94,6 +94,8 @@ class Provider(Plugin): try: data = XMLTree.fromstring(ss(data)) return self.getElements(data, item_path) + except XMLTree.ParseError: + log.error('Invalid XML returned, check "%s" manually for issues', url) except: log.error('Failed to parsing %s: %s', (self.getName(), traceback.format_exc())) From 1827c2e4cd43de2f5e74d6076ab6a8d2520e2af7 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 10 Jan 2015 12:17:30 +0100 Subject: [PATCH 12/60] Don't parse omgwtfnzb if no results are returned --- .../media/_base/providers/nzb/omgwtfnzbs.py | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/couchpotato/core/media/_base/providers/nzb/omgwtfnzbs.py b/couchpotato/core/media/_base/providers/nzb/omgwtfnzbs.py index 21799787..ea5f90f7 100644 --- a/couchpotato/core/media/_base/providers/nzb/omgwtfnzbs.py +++ b/couchpotato/core/media/_base/providers/nzb/omgwtfnzbs.py @@ -44,17 +44,18 @@ class Base(NZBProvider, RSS): nzbs = self.getJsonData(self.urls['search'] % params) - for nzb in nzbs: + if isinstance(nzbs, list): + for nzb in nzbs: - results.append({ - 'id': nzb.get('nzbid'), - 'name': toUnicode(nzb.get('release')), - 'age': self.calculateAge(tryInt(nzb.get('usenetage'))), - 'size': tryInt(nzb.get('sizebytes')) / 1024 / 1024, - 'url': nzb.get('getnzb'), - 'detail_url': nzb.get('details'), - 'description': nzb.get('weblink') - }) + results.append({ + 'id': nzb.get('nzbid'), + 'name': toUnicode(nzb.get('release')), + 'age': self.calculateAge(tryInt(nzb.get('usenetage'))), + 'size': tryInt(nzb.get('sizebytes')) / 1024 / 1024, + 'url': nzb.get('getnzb'), + 'detail_url': nzb.get('details'), + 'description': nzb.get('weblink') + }) config = [{ From 132fa12ef4d5f12493e2c60ae1d2afbd2e67ed1d Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 10 Jan 2015 12:17:47 +0100 Subject: [PATCH 13/60] Late list not loaded on home --- couchpotato/core/media/movie/charts/static/charts.js | 3 ++- couchpotato/core/media/movie/suggestion/static/suggest.js | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/couchpotato/core/media/movie/charts/static/charts.js b/couchpotato/core/media/movie/charts/static/charts.js index 3d70f7f8..d70a1c64 100644 --- a/couchpotato/core/media/movie/charts/static/charts.js +++ b/couchpotato/core/media/movie/charts/static/charts.js @@ -44,11 +44,12 @@ var Charts = new Class({ if( Cookie.read('suggestions_charts_menu_selected') === 'charts'){ self.show(); - self.fireEvent.delay(0, self, 'created'); } else self.el.hide(); + self.fireEvent.delay(0, self, 'created'); + }, fill: function(json){ diff --git a/couchpotato/core/media/movie/suggestion/static/suggest.js b/couchpotato/core/media/movie/suggestion/static/suggest.js index ca4b07c2..ace7f387 100644 --- a/couchpotato/core/media/movie/suggestion/static/suggest.js +++ b/couchpotato/core/media/movie/suggestion/static/suggest.js @@ -51,8 +51,8 @@ var SuggestList = new Class({ self.show(); else self.hide(); - - self.fireEvent('created'); + + self.fireEvent.delay(0, self, 'created'); }, From 12148217a292c2836a8bed28de7c97cf0cf70ec7 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 10 Jan 2015 13:41:17 +0100 Subject: [PATCH 14/60] Log failed notification --- couchpotato/core/notifications/core/main.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/couchpotato/core/notifications/core/main.py b/couchpotato/core/notifications/core/main.py index 771d9696..4c39fb77 100644 --- a/couchpotato/core/notifications/core/main.py +++ b/couchpotato/core/notifications/core/main.py @@ -149,16 +149,15 @@ class CoreNotifier(Notification): def notify(self, message = '', data = None, listener = None): if not data: data = {} + n = { + '_t': 'notification', + 'time': int(time.time()), + } + try: db = get_db() - data['notification_type'] = listener if listener else 'unknown' - - n = { - '_t': 'notification', - 'time': int(time.time()), - 'message': toUnicode(message) - } + n['message'] = toUnicode(message) if data.get('sticky'): n['sticky'] = True @@ -171,7 +170,7 @@ class CoreNotifier(Notification): return True except: - log.error('Failed notify: %s', traceback.format_exc()) + log.error('Failed notify "%s": %s', (n, traceback.format_exc())) def frontend(self, type = 'notification', data = None, message = None): if not data: data = {} From 601f0b54cff4f1768105254e801922d1efacb9a8 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 11 Jan 2015 00:25:51 +0100 Subject: [PATCH 15/60] Send CP header when downloading from newznab --- couchpotato/core/media/_base/providers/nzb/newznab.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/media/_base/providers/nzb/newznab.py b/couchpotato/core/media/_base/providers/nzb/newznab.py index 62b787d8..21ce94b3 100644 --- a/couchpotato/core/media/_base/providers/nzb/newznab.py +++ b/couchpotato/core/media/_base/providers/nzb/newznab.py @@ -183,7 +183,7 @@ class Base(NZBProvider, RSS): return 'try_next' try: - data = self.urlopen(url, show_error = False) + data = self.urlopen(url, show_error = False, headers = {'User-Agent': Env.getIdentifier()}) self.limits_reached[host] = False return data except HTTPError as e: From 17fa33a49608e17cbfd260064cadfaac9d9a8368 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 11 Jan 2015 00:25:58 +0100 Subject: [PATCH 16/60] Update user agent --- couchpotato/core/plugins/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/plugins/base.py b/couchpotato/core/plugins/base.py index 5a90d92b..c02e8f75 100644 --- a/couchpotato/core/plugins/base.py +++ b/couchpotato/core/plugins/base.py @@ -39,7 +39,7 @@ class Plugin(object): _locks = {} - user_agent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:24.0) Gecko/20130519 Firefox/24.0' + user_agent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:34.0) Gecko/20100101 Firefox/34.0' http_last_use = {} http_time_between_calls = 0 http_failed_request = {} From e1bb8c5419c2ac470a1e4051ef5639fe2ec3c440 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 11 Jan 2015 16:15:52 +0100 Subject: [PATCH 17/60] Update Chardet --- libs/chardet/__init__.py | 2 +- libs/chardet/chardetect.py | 66 +++++++++++++++++++++++-------- libs/chardet/jpcntx.py | 8 ++++ libs/chardet/latin1prober.py | 6 +-- libs/chardet/mbcssm.py | 9 ++--- libs/chardet/sjisprober.py | 2 +- libs/chardet/universaldetector.py | 4 +- 7 files changed, 68 insertions(+), 29 deletions(-) diff --git a/libs/chardet/__init__.py b/libs/chardet/__init__.py index e4f0799d..82c2a48d 100755 --- a/libs/chardet/__init__.py +++ b/libs/chardet/__init__.py @@ -15,7 +15,7 @@ # 02110-1301 USA ######################### END LICENSE BLOCK ######################### -__version__ = "2.2.1" +__version__ = "2.3.0" from sys import version_info diff --git a/libs/chardet/chardetect.py b/libs/chardet/chardetect.py index ecd0163b..ffe892f2 100644 --- a/libs/chardet/chardetect.py +++ b/libs/chardet/chardetect.py @@ -12,34 +12,68 @@ Example:: If no paths are provided, it takes its input from stdin. """ -from io import open -from sys import argv, stdin +from __future__ import absolute_import, print_function, unicode_literals + +import argparse +import sys +from io import open + +from chardet import __version__ from chardet.universaldetector import UniversalDetector -def description_of(file, name='stdin'): - """Return a string describing the probable encoding of a file.""" +def description_of(lines, name='stdin'): + """ + Return a string describing the probable encoding of a file or + list of strings. + + :param lines: The lines to get the encoding of. + :type lines: Iterable of bytes + :param name: Name of file or collection of lines + :type name: str + """ u = UniversalDetector() - for line in file: + for line in lines: u.feed(line) u.close() result = u.result if result['encoding']: - return '%s: %s with confidence %s' % (name, - result['encoding'], - result['confidence']) + return '{0}: {1} with confidence {2}'.format(name, result['encoding'], + result['confidence']) else: - return '%s: no result' % name + return '{0}: no result'.format(name) -def main(): - if len(argv) <= 1: - print(description_of(stdin)) - else: - for path in argv[1:]: - with open(path, 'rb') as f: - print(description_of(f, path)) +def main(argv=None): + ''' + Handles command line arguments and gets things started. + + :param argv: List of arguments, as if specified on the command-line. + If None, ``sys.argv[1:]`` is used instead. + :type argv: list of str + ''' + # Get command line arguments + parser = argparse.ArgumentParser( + description="Takes one or more file paths and reports their detected \ + encodings", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + conflict_handler='resolve') + parser.add_argument('input', + help='File whose encoding we would like to determine.', + type=argparse.FileType('rb'), nargs='*', + default=[sys.stdin]) + parser.add_argument('--version', action='version', + version='%(prog)s {0}'.format(__version__)) + args = parser.parse_args(argv) + + for f in args.input: + if f.isatty(): + print("You are running chardetect interactively. Press " + + "CTRL-D twice at the start of a blank line to signal the " + + "end of your input. If you want help, run chardetect " + + "--help\n", file=sys.stderr) + print(description_of(f, f.name)) if __name__ == '__main__': diff --git a/libs/chardet/jpcntx.py b/libs/chardet/jpcntx.py index f7f69ba4..59aeb6a8 100755 --- a/libs/chardet/jpcntx.py +++ b/libs/chardet/jpcntx.py @@ -177,6 +177,12 @@ class JapaneseContextAnalysis: return -1, 1 class SJISContextAnalysis(JapaneseContextAnalysis): + def __init__(self): + self.charset_name = "SHIFT_JIS" + + def get_charset_name(self): + return self.charset_name + def get_order(self, aBuf): if not aBuf: return -1, 1 @@ -184,6 +190,8 @@ class SJISContextAnalysis(JapaneseContextAnalysis): first_char = wrap_ord(aBuf[0]) if ((0x81 <= first_char <= 0x9F) or (0xE0 <= first_char <= 0xFC)): charLen = 2 + if (first_char == 0x87) or (0xFA <= first_char <= 0xFC): + self.charset_name = "CP932" else: charLen = 1 diff --git a/libs/chardet/latin1prober.py b/libs/chardet/latin1prober.py index ad695f57..eef35735 100755 --- a/libs/chardet/latin1prober.py +++ b/libs/chardet/latin1prober.py @@ -129,11 +129,11 @@ class Latin1Prober(CharSetProber): if total < 0.01: confidence = 0.0 else: - confidence = ((self._mFreqCounter[3] / total) - - (self._mFreqCounter[1] * 20.0 / total)) + confidence = ((self._mFreqCounter[3] - self._mFreqCounter[1] * 20.0) + / total) if confidence < 0.0: confidence = 0.0 # lower the confidence of latin1 so that other more accurate # detector can take priority. - confidence = confidence * 0.5 + confidence = confidence * 0.73 return confidence diff --git a/libs/chardet/mbcssm.py b/libs/chardet/mbcssm.py index 3f93cfb0..efe678ca 100755 --- a/libs/chardet/mbcssm.py +++ b/libs/chardet/mbcssm.py @@ -353,7 +353,7 @@ SJIS_cls = ( 2,2,2,2,2,2,2,2, # 68 - 6f 2,2,2,2,2,2,2,2, # 70 - 77 2,2,2,2,2,2,2,1, # 78 - 7f - 3,3,3,3,3,3,3,3, # 80 - 87 + 3,3,3,3,3,2,2,3, # 80 - 87 3,3,3,3,3,3,3,3, # 88 - 8f 3,3,3,3,3,3,3,3, # 90 - 97 3,3,3,3,3,3,3,3, # 98 - 9f @@ -369,9 +369,8 @@ SJIS_cls = ( 2,2,2,2,2,2,2,2, # d8 - df 3,3,3,3,3,3,3,3, # e0 - e7 3,3,3,3,3,4,4,4, # e8 - ef - 4,4,4,4,4,4,4,4, # f0 - f7 - 4,4,4,4,4,0,0,0 # f8 - ff -) + 3,3,3,3,3,3,3,3, # f0 - f7 + 3,3,3,3,3,0,0,0) # f8 - ff SJIS_st = ( @@ -571,5 +570,3 @@ UTF8SMModel = {'classTable': UTF8_cls, 'stateTable': UTF8_st, 'charLenTable': UTF8CharLenTable, 'name': 'UTF-8'} - -# flake8: noqa diff --git a/libs/chardet/sjisprober.py b/libs/chardet/sjisprober.py index b173614e..cd0e9e70 100755 --- a/libs/chardet/sjisprober.py +++ b/libs/chardet/sjisprober.py @@ -47,7 +47,7 @@ class SJISProber(MultiByteCharSetProber): self._mContextAnalyzer.reset() def get_charset_name(self): - return "SHIFT_JIS" + return self._mContextAnalyzer.get_charset_name() def feed(self, aBuf): aLen = len(aBuf) diff --git a/libs/chardet/universaldetector.py b/libs/chardet/universaldetector.py index 9a03ad3d..476522b9 100755 --- a/libs/chardet/universaldetector.py +++ b/libs/chardet/universaldetector.py @@ -71,9 +71,9 @@ class UniversalDetector: if not self._mGotData: # If the data starts with BOM, we know it is UTF - if aBuf[:3] == codecs.BOM: + if aBuf[:3] == codecs.BOM_UTF8: # EF BB BF UTF-8 with BOM - self.result = {'encoding': "UTF-8", 'confidence': 1.0} + self.result = {'encoding': "UTF-8-SIG", 'confidence': 1.0} elif aBuf[:4] == codecs.BOM_UTF32_LE: # FF FE 00 00 UTF-32, little-endian BOM self.result = {'encoding': "UTF-32LE", 'confidence': 1.0} From e1e39cd3f4194d6805cf707063eb0a40206ac883 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 11 Jan 2015 16:17:33 +0100 Subject: [PATCH 18/60] Update requests --- libs/requests/__init__.py | 8 +- libs/requests/adapters.py | 40 ++-- libs/requests/api.py | 21 +- libs/requests/auth.py | 18 +- libs/requests/compat.py | 2 +- libs/requests/exceptions.py | 17 +- libs/requests/models.py | 69 ++++-- libs/requests/packages/urllib3/__init__.py | 2 +- .../requests/packages/urllib3/_collections.py | 7 +- libs/requests/packages/urllib3/connection.py | 24 ++- .../packages/urllib3/connectionpool.py | 64 +++--- .../packages/urllib3/contrib/pyopenssl.py | 25 ++- libs/requests/packages/urllib3/exceptions.py | 13 +- libs/requests/packages/urllib3/request.py | 28 ++- libs/requests/packages/urllib3/util/retry.py | 24 ++- libs/requests/packages/urllib3/util/ssl_.py | 202 ++++++++++++++---- libs/requests/packages/urllib3/util/url.py | 45 +++- libs/requests/sessions.py | 66 ++++-- libs/requests/utils.py | 37 +++- 19 files changed, 521 insertions(+), 191 deletions(-) diff --git a/libs/requests/__init__.py b/libs/requests/__init__.py index 33a2b27a..ac2b06c8 100644 --- a/libs/requests/__init__.py +++ b/libs/requests/__init__.py @@ -13,7 +13,7 @@ Requests is an HTTP library, written in Python, for human beings. Basic GET usage: >>> import requests - >>> r = requests.get('http://python.org') + >>> r = requests.get('https://www.python.org') >>> r.status_code 200 >>> 'Python is a programming language' in r.content @@ -22,7 +22,7 @@ usage: ... or POST: >>> payload = dict(key1='value1', key2='value2') - >>> r = requests.post("http://httpbin.org/post", data=payload) + >>> r = requests.post('http://httpbin.org/post', data=payload) >>> print(r.text) { ... @@ -42,8 +42,8 @@ is at . """ __title__ = 'requests' -__version__ = '2.4.0' -__build__ = 0x020400 +__version__ = '2.5.1' +__build__ = 0x020501 __author__ = 'Kenneth Reitz' __license__ = 'Apache 2.0' __copyright__ = 'Copyright 2014 Kenneth Reitz' diff --git a/libs/requests/adapters.py b/libs/requests/adapters.py index 3c1e979f..c892853b 100644 --- a/libs/requests/adapters.py +++ b/libs/requests/adapters.py @@ -15,19 +15,21 @@ from .packages.urllib3 import Retry from .packages.urllib3.poolmanager import PoolManager, proxy_from_url from .packages.urllib3.response import HTTPResponse from .packages.urllib3.util import Timeout as TimeoutSauce -from .compat import urlparse, basestring, urldefrag +from .compat import urlparse, basestring from .utils import (DEFAULT_CA_BUNDLE_PATH, get_encoding_from_headers, - prepend_scheme_if_needed, get_auth_from_url) + prepend_scheme_if_needed, get_auth_from_url, urldefragauth) from .structures import CaseInsensitiveDict from .packages.urllib3.exceptions import ConnectTimeoutError from .packages.urllib3.exceptions import HTTPError as _HTTPError from .packages.urllib3.exceptions import MaxRetryError from .packages.urllib3.exceptions import ProxyError as _ProxyError +from .packages.urllib3.exceptions import ProtocolError from .packages.urllib3.exceptions import ReadTimeoutError from .packages.urllib3.exceptions import SSLError as _SSLError +from .packages.urllib3.exceptions import ResponseError from .cookies import extract_cookies_to_jar from .exceptions import (ConnectionError, ConnectTimeout, ReadTimeout, SSLError, - ProxyError) + ProxyError, RetryError) from .auth import _basic_auth_str DEFAULT_POOLBLOCK = False @@ -59,8 +61,12 @@ class HTTPAdapter(BaseAdapter): :param pool_connections: The number of urllib3 connection pools to cache. :param pool_maxsize: The maximum number of connections to save in the pool. :param int max_retries: The maximum number of retries each connection - should attempt. Note, this applies only to failed connections and - timeouts, never to requests where the server returns a response. + should attempt. Note, this applies only to failed DNS lookups, socket + connections and connection timeouts, never to requests where data has + made it to the server. By default, Requests does not retry failed + connections. If you need granular control over the conditions under + which we retry a request, import urllib3's ``Retry`` class and pass + that instead. :param pool_block: Whether the connection pool should block for connections. Usage:: @@ -76,7 +82,10 @@ class HTTPAdapter(BaseAdapter): def __init__(self, pool_connections=DEFAULT_POOLSIZE, pool_maxsize=DEFAULT_POOLSIZE, max_retries=DEFAULT_RETRIES, pool_block=DEFAULT_POOLBLOCK): - self.max_retries = max_retries + if max_retries == DEFAULT_RETRIES: + self.max_retries = Retry(0, read=False) + else: + self.max_retries = Retry.from_int(max_retries) self.config = {} self.proxy_manager = {} @@ -122,7 +131,7 @@ class HTTPAdapter(BaseAdapter): self._pool_block = block self.poolmanager = PoolManager(num_pools=connections, maxsize=maxsize, - block=block, **pool_kwargs) + block=block, strict=True, **pool_kwargs) def proxy_manager_for(self, proxy, **proxy_kwargs): """Return urllib3 ProxyManager for the given proxy. @@ -269,7 +278,7 @@ class HTTPAdapter(BaseAdapter): proxy = proxies.get(scheme) if proxy and scheme != 'https': - url, _ = urldefrag(request.url) + url = urldefragauth(request.url) else: url = request.path_url @@ -316,8 +325,10 @@ class HTTPAdapter(BaseAdapter): :param request: The :class:`PreparedRequest ` being sent. :param stream: (optional) Whether to stream the request content. - :param timeout: (optional) The timeout on the request. - :type timeout: float or tuple (connect timeout, read timeout), eg (3.1, 20) + :param timeout: (optional) How long to wait for the server to send + data before giving up, as a float, or a (`connect timeout, read + timeout `_) tuple. + :type timeout: float or tuple :param verify: (optional) Whether to verify SSL certificates. :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. @@ -355,7 +366,7 @@ class HTTPAdapter(BaseAdapter): assert_same_host=False, preload_content=False, decode_content=False, - retries=Retry(self.max_retries, read=False), + retries=self.max_retries, timeout=timeout ) @@ -400,13 +411,16 @@ class HTTPAdapter(BaseAdapter): # All is well, return the connection to the pool. conn._put_conn(low_conn) - except socket.error as sockerr: - raise ConnectionError(sockerr, request=request) + except (ProtocolError, socket.error) as err: + raise ConnectionError(err, request=request) except MaxRetryError as e: if isinstance(e.reason, ConnectTimeoutError): raise ConnectTimeout(e, request=request) + if isinstance(e.reason, ResponseError): + raise RetryError(e, request=request) + raise ConnectionError(e, request=request) except _ProxyError as e: diff --git a/libs/requests/api.py b/libs/requests/api.py index 01d853d5..1469b05c 100644 --- a/libs/requests/api.py +++ b/libs/requests/api.py @@ -22,12 +22,17 @@ def request(method, url, **kwargs): :param url: URL for the new :class:`Request` object. :param params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`. :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. + :param json: (optional) json data to send in the body of the :class:`Request`. :param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`. :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`. - :param files: (optional) Dictionary of 'name': file-like-objects (or {'name': ('filename', fileobj)}) for multipart encoding upload. + :param files: (optional) Dictionary of ``'name': file-like-objects`` (or ``{'name': ('filename', fileobj)}``) for multipart encoding upload. :param auth: (optional) Auth tuple to enable Basic/Digest/Custom HTTP Auth. - :param timeout: (optional) Float describing the timeout of the request in seconds. + :param timeout: (optional) How long to wait for the server to send data + before giving up, as a float, or a (`connect timeout, read timeout + `_) tuple. + :type timeout: float or tuple :param allow_redirects: (optional) Boolean. Set to True if POST/PUT/DELETE redirect following is allowed. + :type allow_redirects: bool :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy. :param verify: (optional) if ``True``, the SSL cert will be verified. A CA_BUNDLE path can also be provided. :param stream: (optional) if ``False``, the response content will be immediately downloaded. @@ -41,7 +46,12 @@ def request(method, url, **kwargs): """ session = sessions.Session() - return session.request(method=method, url=url, **kwargs) + response = session.request(method=method, url=url, **kwargs) + # By explicitly closing the session, we avoid leaving sockets open which + # can trigger a ResourceWarning in some cases, and look like a memory leak + # in others. + session.close() + return response def get(url, **kwargs): @@ -77,15 +87,16 @@ def head(url, **kwargs): return request('head', url, **kwargs) -def post(url, data=None, **kwargs): +def post(url, data=None, json=None, **kwargs): """Sends a POST request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. + :param json: (optional) json data to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. """ - return request('post', url, data=data, **kwargs) + return request('post', url, data=data, json=json, **kwargs) def put(url, data=None, **kwargs): diff --git a/libs/requests/auth.py b/libs/requests/auth.py index 9b6426dc..b950181d 100644 --- a/libs/requests/auth.py +++ b/libs/requests/auth.py @@ -17,6 +17,7 @@ from base64 import b64encode from .compat import urlparse, str from .cookies import extract_cookies_to_jar from .utils import parse_dict_header, to_native_string +from .status_codes import codes CONTENT_TYPE_FORM_URLENCODED = 'application/x-www-form-urlencoded' CONTENT_TYPE_MULTI_PART = 'multipart/form-data' @@ -66,6 +67,7 @@ class HTTPDigestAuth(AuthBase): self.nonce_count = 0 self.chal = {} self.pos = None + self.num_401_calls = 1 def build_digest_header(self, method, url): @@ -150,6 +152,11 @@ class HTTPDigestAuth(AuthBase): return 'Digest %s' % (base) + def handle_redirect(self, r, **kwargs): + """Reset num_401_calls counter on redirects.""" + if r.is_redirect: + self.num_401_calls = 1 + def handle_401(self, r, **kwargs): """Takes the given response and tries digest-auth, if needed.""" @@ -162,7 +169,7 @@ class HTTPDigestAuth(AuthBase): if 'digest' in s_auth.lower() and num_401_calls < 2: - setattr(self, 'num_401_calls', num_401_calls + 1) + self.num_401_calls += 1 pat = re.compile(r'digest ', flags=re.IGNORECASE) self.chal = parse_dict_header(pat.sub('', s_auth, count=1)) @@ -182,7 +189,7 @@ class HTTPDigestAuth(AuthBase): return _r - setattr(self, 'num_401_calls', 1) + self.num_401_calls = 1 return r def __call__(self, r): @@ -192,6 +199,11 @@ class HTTPDigestAuth(AuthBase): try: self.pos = r.body.tell() except AttributeError: - pass + # In the case of HTTPDigestAuth being reused and the body of + # the previous request was a file-like object, pos has the + # file position of the previous body. Ensure it's set to + # None. + self.pos = None r.register_hook('response', self.handle_401) + r.register_hook('response', self.handle_redirect) return r diff --git a/libs/requests/compat.py b/libs/requests/compat.py index 439b16f5..a294a329 100644 --- a/libs/requests/compat.py +++ b/libs/requests/compat.py @@ -76,7 +76,7 @@ is_solaris = ('solar==' in str(sys.platform).lower()) # Complete guess. try: import simplejson as json except (ImportError, SyntaxError): - # simplejson does not support Python 3.2, it thows a SyntaxError + # simplejson does not support Python 3.2, it throws a SyntaxError # because of u'...' Unicode literals. import json diff --git a/libs/requests/exceptions.py b/libs/requests/exceptions.py index 6dbd98a9..89135a80 100644 --- a/libs/requests/exceptions.py +++ b/libs/requests/exceptions.py @@ -46,15 +46,16 @@ class SSLError(ConnectionError): class Timeout(RequestException): """The request timed out. - Catching this error will catch both :exc:`ConnectTimeout` and - :exc:`ReadTimeout` errors. + Catching this error will catch both + :exc:`~requests.exceptions.ConnectTimeout` and + :exc:`~requests.exceptions.ReadTimeout` errors. """ class ConnectTimeout(ConnectionError, Timeout): - """The request timed out while trying to connect to the server. + """The request timed out while trying to connect to the remote server. - Requests that produce this error are safe to retry + Requests that produced this error are safe to retry. """ @@ -88,3 +89,11 @@ class ChunkedEncodingError(RequestException): class ContentDecodingError(RequestException, BaseHTTPError): """Failed to decode response content""" + + +class StreamConsumedError(RequestException, TypeError): + """The content for this response was already consumed""" + + +class RetryError(RequestException): + """Custom retries logic failed""" diff --git a/libs/requests/models.py b/libs/requests/models.py index 03ff627a..b728c84e 100644 --- a/libs/requests/models.py +++ b/libs/requests/models.py @@ -20,10 +20,10 @@ from .packages.urllib3.fields import RequestField from .packages.urllib3.filepost import encode_multipart_formdata from .packages.urllib3.util import parse_url from .packages.urllib3.exceptions import ( - DecodeError, ReadTimeoutError, ProtocolError) + DecodeError, ReadTimeoutError, ProtocolError, LocationParseError) from .exceptions import ( - HTTPError, RequestException, MissingSchema, InvalidURL, - ChunkedEncodingError, ContentDecodingError, ConnectionError) + HTTPError, MissingSchema, InvalidURL, ChunkedEncodingError, + ContentDecodingError, ConnectionError, StreamConsumedError) from .utils import ( guess_filename, get_auth_from_url, requote_uri, stream_decode_response_unicode, to_key_val_list, parse_header_links, @@ -46,6 +46,8 @@ DEFAULT_REDIRECT_LIMIT = 30 CONTENT_CHUNK_SIZE = 10 * 1024 ITER_CHUNK_SIZE = 512 +json_dumps = json.dumps + class RequestEncodingMixin(object): @property @@ -189,7 +191,8 @@ class Request(RequestHooksMixin): :param url: URL to send. :param headers: dictionary of headers to send. :param files: dictionary of {filename: fileobject} files to multipart upload. - :param data: the body to attach the request. If a dictionary is provided, form-encoding will take place. + :param data: the body to attach to the request. If a dictionary is provided, form-encoding will take place. + :param json: json for the body to attach to the request (if data is not specified). :param params: dictionary of URL parameters to append to the URL. :param auth: Auth handler or (user, pass) tuple. :param cookies: dictionary or CookieJar of cookies to attach to this request. @@ -212,7 +215,8 @@ class Request(RequestHooksMixin): params=None, auth=None, cookies=None, - hooks=None): + hooks=None, + json=None): # Default empty dicts for dict params. data = [] if data is None else data @@ -230,6 +234,7 @@ class Request(RequestHooksMixin): self.headers = headers self.files = files self.data = data + self.json = json self.params = params self.auth = auth self.cookies = cookies @@ -246,6 +251,7 @@ class Request(RequestHooksMixin): headers=self.headers, files=self.files, data=self.data, + json=self.json, params=self.params, auth=self.auth, cookies=self.cookies, @@ -289,14 +295,15 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): self.hooks = default_hooks() def prepare(self, method=None, url=None, headers=None, files=None, - data=None, params=None, auth=None, cookies=None, hooks=None): + data=None, params=None, auth=None, cookies=None, hooks=None, + json=None): """Prepares the entire request with the given parameters.""" self.prepare_method(method) self.prepare_url(url, params) self.prepare_headers(headers) self.prepare_cookies(cookies) - self.prepare_body(data, files) + self.prepare_body(data, files, json) self.prepare_auth(auth, url) # Note that prepare_auth must be last to enable authentication schemes # such as OAuth to work on a fully prepared request. @@ -326,21 +333,27 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): def prepare_url(self, url, params): """Prepares the given HTTP URL.""" #: Accept objects that have string representations. - try: - url = unicode(url) - except NameError: - # We're on Python 3. - url = str(url) - except UnicodeDecodeError: - pass + #: We're unable to blindy call unicode/str functions + #: as this will include the bytestring indicator (b'') + #: on python 3.x. + #: https://github.com/kennethreitz/requests/pull/2238 + if isinstance(url, bytes): + url = url.decode('utf8') + else: + url = unicode(url) if is_py2 else str(url) - # Don't do any URL preparation for oddball schemes + # Don't do any URL preparation for non-HTTP schemes like `mailto`, + # `data` etc to work around exceptions from `url_parse`, which + # handles RFC 3986 only. if ':' in url and not url.lower().startswith('http'): self.url = url return # Support for unicode domain names and paths. - scheme, auth, host, port, path, query, fragment = parse_url(url) + try: + scheme, auth, host, port, path, query, fragment = parse_url(url) + except LocationParseError as e: + raise InvalidURL(*e.args) if not scheme: raise MissingSchema("Invalid URL {0!r}: No schema supplied. " @@ -397,7 +410,7 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): else: self.headers = CaseInsensitiveDict() - def prepare_body(self, data, files): + def prepare_body(self, data, files, json=None): """Prepares the given HTTP body data.""" # Check if file, fo, generator, iterator. @@ -408,6 +421,10 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): content_type = None length = None + if json is not None: + content_type = 'application/json' + body = json_dumps(json) + is_stream = all([ hasattr(data, '__iter__'), not isinstance(data, (basestring, list, tuple, dict)) @@ -433,7 +450,7 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): if files: (body, content_type) = self._encode_files(files, data) else: - if data: + if data and json is None: body = self._encode_params(data) if isinstance(data, basestring) or hasattr(data, 'read'): content_type = None @@ -443,7 +460,7 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): self.prepare_content_length(body) # Add content-type if it wasn't explicitly provided. - if (content_type) and (not 'content-type' in self.headers): + if content_type and ('content-type' not in self.headers): self.headers['Content-Type'] = content_type self.body = body @@ -457,7 +474,7 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): l = super_len(body) if l: self.headers['Content-Length'] = builtin_str(l) - elif self.method not in ('GET', 'HEAD'): + elif (self.method not in ('GET', 'HEAD')) and (self.headers.get('Content-Length') is None): self.headers['Content-Length'] = '0' def prepare_auth(self, auth, url=''): @@ -600,7 +617,7 @@ class Response(object): def ok(self): try: self.raise_for_status() - except RequestException: + except HTTPError: return False return True @@ -653,6 +670,8 @@ class Response(object): self._content_consumed = True + if self._content_consumed and isinstance(self._content, bool): + raise StreamConsumedError() # simulate reading small chunks of the content reused_chunks = iter_slices(self._content, chunk_size) @@ -665,7 +684,7 @@ class Response(object): return chunks - def iter_lines(self, chunk_size=ITER_CHUNK_SIZE, decode_unicode=None): + def iter_lines(self, chunk_size=ITER_CHUNK_SIZE, decode_unicode=None, delimiter=None): """Iterates over the response data, one line at a time. When stream=True is set on the request, this avoids reading the content at once into memory for large responses. @@ -677,7 +696,11 @@ class Response(object): if pending is not None: chunk = pending + chunk - lines = chunk.splitlines() + + if delimiter: + lines = chunk.split(delimiter) + else: + lines = chunk.splitlines() if lines and lines[-1] and chunk and lines[-1][-1] == chunk[-1]: pending = lines.pop() diff --git a/libs/requests/packages/urllib3/__init__.py b/libs/requests/packages/urllib3/__init__.py index 4b36b5ae..dfc82d03 100644 --- a/libs/requests/packages/urllib3/__init__.py +++ b/libs/requests/packages/urllib3/__init__.py @@ -57,7 +57,7 @@ del NullHandler # Set security warning to only go off once by default. import warnings -warnings.simplefilter('module', exceptions.SecurityWarning) +warnings.simplefilter('always', exceptions.SecurityWarning) def disable_warnings(category=exceptions.HTTPWarning): """ diff --git a/libs/requests/packages/urllib3/_collections.py b/libs/requests/packages/urllib3/_collections.py index d77ebb8d..784342a4 100644 --- a/libs/requests/packages/urllib3/_collections.py +++ b/libs/requests/packages/urllib3/_collections.py @@ -14,7 +14,7 @@ try: # Python 2.7+ from collections import OrderedDict except ImportError: from .packages.ordered_dict import OrderedDict -from .packages.six import itervalues +from .packages.six import iterkeys, itervalues __all__ = ['RecentlyUsedContainer', 'HTTPHeaderDict'] @@ -85,8 +85,7 @@ class RecentlyUsedContainer(MutableMapping): def clear(self): with self.lock: # Copy pointers to all values, then wipe the mapping - # under Python 2, this copies the list of values twice :-| - values = list(self._container.values()) + values = list(itervalues(self._container)) self._container.clear() if self.dispose_func: @@ -95,7 +94,7 @@ class RecentlyUsedContainer(MutableMapping): def keys(self): with self.lock: - return self._container.keys() + return list(iterkeys(self._container)) class HTTPHeaderDict(MutableMapping): diff --git a/libs/requests/packages/urllib3/connection.py b/libs/requests/packages/urllib3/connection.py index c6e1959a..e5de769d 100644 --- a/libs/requests/packages/urllib3/connection.py +++ b/libs/requests/packages/urllib3/connection.py @@ -3,6 +3,7 @@ import sys import socket from socket import timeout as SocketTimeout import warnings +from .packages import six try: # Python 3 from http.client import HTTPConnection as _HTTPConnection, HTTPException @@ -26,12 +27,20 @@ except (ImportError, AttributeError): # Platform-specific: No SSL. pass +try: # Python 3: + # Not a no-op, we're adding this to the namespace so it can be imported. + ConnectionError = ConnectionError +except NameError: # Python 2: + class ConnectionError(Exception): + pass + + from .exceptions import ( ConnectTimeoutError, SystemTimeWarning, + SecurityWarning, ) from .packages.ssl_match_hostname import match_hostname -from .packages import six from .util.ssl_ import ( resolve_cert_reqs, @@ -40,8 +49,8 @@ from .util.ssl_ import ( assert_fingerprint, ) -from .util import connection +from .util import connection port_by_scheme = { 'http': 80, @@ -233,8 +242,15 @@ class VerifiedHTTPSConnection(HTTPSConnection): self.assert_fingerprint) elif resolved_cert_reqs != ssl.CERT_NONE \ and self.assert_hostname is not False: - match_hostname(self.sock.getpeercert(), - self.assert_hostname or hostname) + cert = self.sock.getpeercert() + if not cert.get('subjectAltName', ()): + warnings.warn(( + 'Certificate has no `subjectAltName`, falling back to check for a `commonName` for now. ' + 'This feature is being removed by major browsers and deprecated by RFC 2818. ' + '(See https://github.com/shazow/urllib3/issues/497 for details.)'), + SecurityWarning + ) + match_hostname(cert, self.assert_hostname or hostname) self.is_verified = (resolved_cert_reqs == ssl.CERT_REQUIRED or self.assert_fingerprint is not None) diff --git a/libs/requests/packages/urllib3/connectionpool.py b/libs/requests/packages/urllib3/connectionpool.py index 9cc2a955..70ee4eed 100644 --- a/libs/requests/packages/urllib3/connectionpool.py +++ b/libs/requests/packages/urllib3/connectionpool.py @@ -32,7 +32,7 @@ from .connection import ( port_by_scheme, DummyConnection, HTTPConnection, HTTPSConnection, VerifiedHTTPSConnection, - HTTPException, BaseSSLError, + HTTPException, BaseSSLError, ConnectionError ) from .request import RequestMethods from .response import HTTPResponse @@ -278,6 +278,23 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): # can be removed later return Timeout.from_float(timeout) + def _raise_timeout(self, err, url, timeout_value): + """Is the error actually a timeout? Will raise a ReadTimeout or pass""" + + if isinstance(err, SocketTimeout): + raise ReadTimeoutError(self, url, "Read timed out. (read timeout=%s)" % timeout_value) + + # See the above comment about EAGAIN in Python 3. In Python 2 we have + # to specifically catch it and throw the timeout error + if hasattr(err, 'errno') and err.errno in _blocking_errnos: + raise ReadTimeoutError(self, url, "Read timed out. (read timeout=%s)" % timeout_value) + + # Catch possible read timeouts thrown as SSL errors. If not the + # case, rethrow the original. We need to do this because of: + # http://bugs.python.org/issue10272 + if 'timed out' in str(err) or 'did not complete (read)' in str(err): # Python 2.6 + raise ReadTimeoutError(self, url, "Read timed out. (read timeout=%s)" % timeout_value) + def _make_request(self, conn, method, url, timeout=_Default, **httplib_request_kw): """ @@ -301,7 +318,12 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): conn.timeout = timeout_obj.connect_timeout # Trigger any extra validation we need to do. - self._validate_conn(conn) + try: + self._validate_conn(conn) + except (SocketTimeout, BaseSSLError) as e: + # Py2 raises this as a BaseSSLError, Py3 raises it as socket timeout. + self._raise_timeout(err=e, url=url, timeout_value=conn.timeout) + raise # conn.request() calls httplib.*.request, not the method in # urllib3.request. It also calls makefile (recv) on the socket. @@ -331,28 +353,8 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): httplib_response = conn.getresponse(buffering=True) except TypeError: # Python 2.6 and older httplib_response = conn.getresponse() - except SocketTimeout: - raise ReadTimeoutError( - self, url, "Read timed out. (read timeout=%s)" % read_timeout) - - except BaseSSLError as e: - # Catch possible read timeouts thrown as SSL errors. If not the - # case, rethrow the original. We need to do this because of: - # http://bugs.python.org/issue10272 - if 'timed out' in str(e) or \ - 'did not complete (read)' in str(e): # Python 2.6 - raise ReadTimeoutError( - self, url, "Read timed out. (read timeout=%s)" % read_timeout) - - raise - - except SocketError as e: # Platform-specific: Python 2 - # See the above comment about EAGAIN in Python 3. In Python 2 we - # have to specifically catch it and throw the timeout error - if e.errno in _blocking_errnos: - raise ReadTimeoutError( - self, url, "Read timed out. (read timeout=%s)" % read_timeout) - + except (SocketTimeout, BaseSSLError, SocketError) as e: + self._raise_timeout(err=e, url=url, timeout_value=read_timeout) raise # AppEngine doesn't have a version attr. @@ -537,12 +539,15 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): raise EmptyPoolError(self, "No pool connections are available.") except (BaseSSLError, CertificateError) as e: - # Release connection unconditionally because there is no way to - # close it externally in case of exception. - release_conn = True + # Close the connection. If a connection is reused on which there + # was a Certificate error, the next request will certainly raise + # another Certificate error. + if conn: + conn.close() + conn = None raise SSLError(e) - except (TimeoutError, HTTPException, SocketError) as e: + except (TimeoutError, HTTPException, SocketError, ConnectionError) as e: if conn: # Discard the connection for these exceptions. It will be # be replaced during the next _get_conn() call. @@ -725,8 +730,7 @@ class HTTPSConnectionPool(HTTPConnectionPool): warnings.warn(( 'Unverified HTTPS request is being made. ' 'Adding certificate verification is strongly advised. See: ' - 'https://urllib3.readthedocs.org/en/latest/security.html ' - '(This warning will only appear once by default.)'), + 'https://urllib3.readthedocs.org/en/latest/security.html'), InsecureRequestWarning) diff --git a/libs/requests/packages/urllib3/contrib/pyopenssl.py b/libs/requests/packages/urllib3/contrib/pyopenssl.py index 24de9e40..8229090c 100644 --- a/libs/requests/packages/urllib3/contrib/pyopenssl.py +++ b/libs/requests/packages/urllib3/contrib/pyopenssl.py @@ -29,7 +29,7 @@ Now you can use :mod:`urllib3` as you normally would, and it will support SNI when the required modules are installed. Activating this module also has the positive side effect of disabling SSL/TLS -encryption in Python 2 (see `CRIME attack`_). +compression in Python 2 (see `CRIME attack`_). If you want to configure the default list of supported cipher suites, you can set the ``urllib3.contrib.pyopenssl.DEFAULT_SSL_CIPHER_LIST`` variable. @@ -70,9 +70,14 @@ HAS_SNI = SUBJ_ALT_NAME_SUPPORT # Map from urllib3 to PyOpenSSL compatible parameter-values. _openssl_versions = { ssl.PROTOCOL_SSLv23: OpenSSL.SSL.SSLv23_METHOD, - ssl.PROTOCOL_SSLv3: OpenSSL.SSL.SSLv3_METHOD, ssl.PROTOCOL_TLSv1: OpenSSL.SSL.TLSv1_METHOD, } + +try: + _openssl_versions.update({ssl.PROTOCOL_SSLv3: OpenSSL.SSL.SSLv3_METHOD}) +except AttributeError: + pass + _openssl_verify = { ssl.CERT_NONE: OpenSSL.SSL.VERIFY_NONE, ssl.CERT_OPTIONAL: OpenSSL.SSL.VERIFY_PEER, @@ -199,8 +204,21 @@ class WrappedSocket(object): def settimeout(self, timeout): return self.socket.settimeout(timeout) + def _send_until_done(self, data): + while True: + try: + return self.connection.send(data) + except OpenSSL.SSL.WantWriteError: + _, wlist, _ = select.select([], [self.socket], [], + self.socket.gettimeout()) + if not wlist: + raise timeout() + continue + def sendall(self, data): - return self.connection.sendall(data) + while len(data): + sent = self._send_until_done(data) + data = data[sent:] def close(self): if self._makefile_refs < 1: @@ -248,6 +266,7 @@ def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, ssl_version=None): ctx = OpenSSL.SSL.Context(_openssl_versions[ssl_version]) if certfile: + keyfile = keyfile or certfile # Match behaviour of the normal python ssl library ctx.use_certificate_file(certfile) if keyfile: ctx.use_privatekey_file(keyfile) diff --git a/libs/requests/packages/urllib3/exceptions.py b/libs/requests/packages/urllib3/exceptions.py index 7519ba98..0c6fd3c5 100644 --- a/libs/requests/packages/urllib3/exceptions.py +++ b/libs/requests/packages/urllib3/exceptions.py @@ -72,11 +72,8 @@ class MaxRetryError(RequestError): def __init__(self, pool, url, reason=None): self.reason = reason - message = "Max retries exceeded with url: %s" % url - if reason: - message += " (Caused by %r)" % reason - else: - message += " (Caused by redirect)" + message = "Max retries exceeded with url: %s (Caused by %r)" % ( + url, reason) RequestError.__init__(self, pool, url, message) @@ -141,6 +138,12 @@ class LocationParseError(LocationValueError): self.location = location +class ResponseError(HTTPError): + "Used as a container for an error reason supplied in a MaxRetryError." + GENERIC_ERROR = 'too many error responses' + SPECIFIC_ERROR = 'too many {status_code} error responses' + + class SecurityWarning(HTTPWarning): "Warned when perfoming security reducing actions" pass diff --git a/libs/requests/packages/urllib3/request.py b/libs/requests/packages/urllib3/request.py index 51fe2386..b08d6c92 100644 --- a/libs/requests/packages/urllib3/request.py +++ b/libs/requests/packages/urllib3/request.py @@ -118,18 +118,24 @@ class RequestMethods(object): which is used to compose the body of the request. The random boundary string can be explicitly set with the ``multipart_boundary`` parameter. """ - if encode_multipart: - body, content_type = encode_multipart_formdata( - fields or {}, boundary=multipart_boundary) - else: - body, content_type = (urlencode(fields or {}), - 'application/x-www-form-urlencoded') - if headers is None: headers = self.headers - headers_ = {'Content-Type': content_type} - headers_.update(headers) + extra_kw = {'headers': {}} - return self.urlopen(method, url, body=body, headers=headers_, - **urlopen_kw) + if fields: + if 'body' in urlopen_kw: + raise TypeError('request got values for both \'fields\' and \'body\', can only specify one.') + + if encode_multipart: + body, content_type = encode_multipart_formdata(fields, boundary=multipart_boundary) + else: + body, content_type = urlencode(fields), 'application/x-www-form-urlencoded' + + extra_kw['body'] = body + extra_kw['headers'] = {'Content-Type': content_type} + + extra_kw['headers'].update(headers) + extra_kw.update(urlopen_kw) + + return self.urlopen(method, url, **extra_kw) diff --git a/libs/requests/packages/urllib3/util/retry.py b/libs/requests/packages/urllib3/util/retry.py index eb560dfc..aeaf8a02 100644 --- a/libs/requests/packages/urllib3/util/retry.py +++ b/libs/requests/packages/urllib3/util/retry.py @@ -2,10 +2,11 @@ import time import logging from ..exceptions import ( - ProtocolError, ConnectTimeoutError, - ReadTimeoutError, MaxRetryError, + ProtocolError, + ReadTimeoutError, + ResponseError, ) from ..packages import six @@ -36,7 +37,6 @@ class Retry(object): Errors will be wrapped in :class:`~urllib3.exceptions.MaxRetryError` unless retries are disabled, in which case the causing exception will be raised. - :param int total: Total number of retries to allow. Takes precedence over other counts. @@ -184,8 +184,8 @@ class Retry(object): return isinstance(err, ConnectTimeoutError) def _is_read_error(self, err): - """ Errors that occur after the request has been started, so we can't - assume that the server did not process any of it. + """ Errors that occur after the request has been started, so we should + assume that the server began processing it. """ return isinstance(err, (ReadTimeoutError, ProtocolError)) @@ -198,8 +198,7 @@ class Retry(object): return self.status_forcelist and status_code in self.status_forcelist def is_exhausted(self): - """ Are we out of retries? - """ + """ Are we out of retries? """ retry_counts = (self.total, self.connect, self.read, self.redirect) retry_counts = list(filter(None, retry_counts)) if not retry_counts: @@ -230,6 +229,7 @@ class Retry(object): connect = self.connect read = self.read redirect = self.redirect + cause = 'unknown' if error and self._is_connection_error(error): # Connect retry? @@ -251,10 +251,16 @@ class Retry(object): # Redirect retry? if redirect is not None: redirect -= 1 + cause = 'too many redirects' else: - # FIXME: Nothing changed, scenario doesn't make sense. + # Incrementing because of a server error like a 500 in + # status_forcelist and a the given method is in the whitelist _observed_errors += 1 + cause = ResponseError.GENERIC_ERROR + if response and response.status: + cause = ResponseError.SPECIFIC_ERROR.format( + status_code=response.status) new_retry = self.new( total=total, @@ -262,7 +268,7 @@ class Retry(object): _observed_errors=_observed_errors) if new_retry.is_exhausted(): - raise MaxRetryError(_pool, url, error) + raise MaxRetryError(_pool, url, error or ResponseError(cause)) log.debug("Incremented Retry for (url='%s'): %r" % (url, new_retry)) diff --git a/libs/requests/packages/urllib3/util/ssl_.py b/libs/requests/packages/urllib3/util/ssl_.py index 9cfe2d2a..a788b1b9 100644 --- a/libs/requests/packages/urllib3/util/ssl_.py +++ b/libs/requests/packages/urllib3/util/ssl_.py @@ -4,18 +4,84 @@ from hashlib import md5, sha1 from ..exceptions import SSLError -try: # Test for SSL features - SSLContext = None - HAS_SNI = False +SSLContext = None +HAS_SNI = False +create_default_context = None - import ssl +import errno +import ssl + +try: # Test for SSL features from ssl import wrap_socket, CERT_NONE, PROTOCOL_SSLv23 - from ssl import SSLContext # Modern SSL? from ssl import HAS_SNI # Has SNI? except ImportError: pass +try: + from ssl import OP_NO_SSLv2, OP_NO_SSLv3, OP_NO_COMPRESSION +except ImportError: + OP_NO_SSLv2, OP_NO_SSLv3 = 0x1000000, 0x2000000 + OP_NO_COMPRESSION = 0x20000 + +try: + from ssl import _DEFAULT_CIPHERS +except ImportError: + _DEFAULT_CIPHERS = ( + 'ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+HIGH:' + 'DH+HIGH:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+HIGH:RSA+3DES:ECDH+RC4:' + 'DH+RC4:RSA+RC4:!aNULL:!eNULL:!MD5' + ) + +try: + from ssl import SSLContext # Modern SSL? +except ImportError: + import sys + + class SSLContext(object): # Platform-specific: Python 2 & 3.1 + supports_set_ciphers = sys.version_info >= (2, 7) + + def __init__(self, protocol_version): + self.protocol = protocol_version + # Use default values from a real SSLContext + self.check_hostname = False + self.verify_mode = ssl.CERT_NONE + self.ca_certs = None + self.options = 0 + self.certfile = None + self.keyfile = None + self.ciphers = None + + def load_cert_chain(self, certfile, keyfile): + self.certfile = certfile + self.keyfile = keyfile + + def load_verify_locations(self, location): + self.ca_certs = location + + def set_ciphers(self, cipher_suite): + if not self.supports_set_ciphers: + raise TypeError( + 'Your version of Python does not support setting ' + 'a custom cipher suite. Please upgrade to Python ' + '2.7, 3.2, or later if you need this functionality.' + ) + self.ciphers = cipher_suite + + def wrap_socket(self, socket, server_hostname=None): + kwargs = { + 'keyfile': self.keyfile, + 'certfile': self.certfile, + 'ca_certs': self.ca_certs, + 'cert_reqs': self.verify_mode, + 'ssl_version': self.protocol, + } + if self.supports_set_ciphers: # Platform-specific: Python 2.7+ + return wrap_socket(socket, ciphers=self.ciphers, **kwargs) + else: # Platform-specific: Python 2.6 + return wrap_socket(socket, **kwargs) + + def assert_fingerprint(cert, fingerprint): """ Checks if given fingerprint matches the supplied certificate. @@ -91,42 +157,98 @@ def resolve_ssl_version(candidate): return candidate -if SSLContext is not None: # Python 3.2+ - def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, - ca_certs=None, server_hostname=None, - ssl_version=None): - """ - All arguments except `server_hostname` have the same meaning as for - :func:`ssl.wrap_socket` +def create_urllib3_context(ssl_version=None, cert_reqs=ssl.CERT_REQUIRED, + options=None, ciphers=None): + """All arguments have the same meaning as ``ssl_wrap_socket``. - :param server_hostname: - Hostname of the expected certificate - """ - context = SSLContext(ssl_version) - context.verify_mode = cert_reqs + By default, this function does a lot of the same work that + ``ssl.create_default_context`` does on Python 3.4+. It: - # Disable TLS compression to migitate CRIME attack (issue #309) - OP_NO_COMPRESSION = 0x20000 - context.options |= OP_NO_COMPRESSION + - Disables SSLv2, SSLv3, and compression + - Sets a restricted set of server ciphers - if ca_certs: - try: - context.load_verify_locations(ca_certs) - # Py32 raises IOError - # Py33 raises FileNotFoundError - except Exception as e: # Reraise as SSLError + If you wish to enable SSLv3, you can do:: + + from urllib3.util import ssl_ + context = ssl_.create_urllib3_context() + context.options &= ~ssl_.OP_NO_SSLv3 + + You can do the same to enable compression (substituting ``COMPRESSION`` + for ``SSLv3`` in the last line above). + + :param ssl_version: + The desired protocol version to use. This will default to + PROTOCOL_SSLv23 which will negotiate the highest protocol that both + the server and your installation of OpenSSL support. + :param cert_reqs: + Whether to require the certificate verification. This defaults to + ``ssl.CERT_REQUIRED``. + :param options: + Specific OpenSSL options. These default to ``ssl.OP_NO_SSLv2``, + ``ssl.OP_NO_SSLv3``, ``ssl.OP_NO_COMPRESSION``. + :param ciphers: + Which cipher suites to allow the server to select. + :returns: + Constructed SSLContext object with specified options + :rtype: SSLContext + """ + context = SSLContext(ssl_version or ssl.PROTOCOL_SSLv23) + + if options is None: + options = 0 + # SSLv2 is easily broken and is considered harmful and dangerous + options |= OP_NO_SSLv2 + # SSLv3 has several problems and is now dangerous + options |= OP_NO_SSLv3 + # Disable compression to prevent CRIME attacks for OpenSSL 1.0+ + # (issue #309) + options |= OP_NO_COMPRESSION + + context.options |= options + + if getattr(context, 'supports_set_ciphers', True): # Platform-specific: Python 2.6 + context.set_ciphers(ciphers or _DEFAULT_CIPHERS) + + context.verify_mode = cert_reqs + if getattr(context, 'check_hostname', None) is not None: # Platform-specific: Python 3.2 + context.check_hostname = (context.verify_mode == ssl.CERT_REQUIRED) + return context + + +def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, + ca_certs=None, server_hostname=None, + ssl_version=None, ciphers=None, ssl_context=None): + """ + All arguments except for server_hostname and ssl_context have the same + meaning as they do when using :func:`ssl.wrap_socket`. + + :param server_hostname: + When SNI is supported, the expected hostname of the certificate + :param ssl_context: + A pre-made :class:`SSLContext` object. If none is provided, one will + be created using :func:`create_urllib3_context`. + :param ciphers: + A string of ciphers we wish the client to support. This is not + supported on Python 2.6 as the ssl module does not support it. + """ + context = ssl_context + if context is None: + context = create_urllib3_context(ssl_version, cert_reqs, + ciphers=ciphers) + + if ca_certs: + try: + context.load_verify_locations(ca_certs) + except IOError as e: # Platform-specific: Python 2.6, 2.7, 3.2 + raise SSLError(e) + # Py33 raises FileNotFoundError which subclasses OSError + # These are not equivalent unless we check the errno attribute + except OSError as e: # Platform-specific: Python 3.3 and beyond + if e.errno == errno.ENOENT: raise SSLError(e) - if certfile: - # FIXME: This block needs a test. - context.load_cert_chain(certfile, keyfile) - if HAS_SNI: # Platform-specific: OpenSSL with enabled SNI - return context.wrap_socket(sock, server_hostname=server_hostname) - return context.wrap_socket(sock) - -else: # Python 3.1 and earlier - def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, - ca_certs=None, server_hostname=None, - ssl_version=None): - return wrap_socket(sock, keyfile=keyfile, certfile=certfile, - ca_certs=ca_certs, cert_reqs=cert_reqs, - ssl_version=ssl_version) + raise + if certfile: + context.load_cert_chain(certfile, keyfile) + if HAS_SNI: # Platform-specific: OpenSSL with enabled SNI + return context.wrap_socket(sock, server_hostname=server_hostname) + return context.wrap_socket(sock) diff --git a/libs/requests/packages/urllib3/util/url.py b/libs/requests/packages/urllib3/util/url.py index 487d456c..b2ec834f 100644 --- a/libs/requests/packages/urllib3/util/url.py +++ b/libs/requests/packages/urllib3/util/url.py @@ -40,6 +40,48 @@ class Url(namedtuple('Url', url_attrs)): return '%s:%d' % (self.host, self.port) return self.host + @property + def url(self): + """ + Convert self into a url + + This function should more or less round-trip with :func:`.parse_url`. The + returned url may not be exactly the same as the url inputted to + :func:`.parse_url`, but it should be equivalent by the RFC (e.g., urls + with a blank port will have : removed). + + Example: :: + + >>> U = parse_url('http://google.com/mail/') + >>> U.url + 'http://google.com/mail/' + >>> Url('http', 'username:password', 'host.com', 80, + ... '/path', 'query', 'fragment').url + 'http://username:password@host.com:80/path?query#fragment' + """ + scheme, auth, host, port, path, query, fragment = self + url = '' + + # We use "is not None" we want things to happen with empty strings (or 0 port) + if scheme is not None: + url += scheme + '://' + if auth is not None: + url += auth + '@' + if host is not None: + url += host + if port is not None: + url += ':' + str(port) + if path is not None: + url += path + if query is not None: + url += '?' + query + if fragment is not None: + url += '#' + fragment + + return url + + def __str__(self): + return self.url def split_first(s, delims): """ @@ -84,7 +126,7 @@ def parse_url(url): Example:: >>> parse_url('http://google.com/mail/') - Url(scheme='http', host='google.com', port=None, path='/', ...) + Url(scheme='http', host='google.com', port=None, path='/mail/', ...) >>> parse_url('google.com:80') Url(scheme=None, host='google.com', port=80, path=None, ...) >>> parse_url('/foo?bar') @@ -162,7 +204,6 @@ def parse_url(url): return Url(scheme, auth, host, port, path, query, fragment) - def get_host(url): """ Deprecated. Use :func:`.parse_url` instead. diff --git a/libs/requests/sessions.py b/libs/requests/sessions.py index 02c9fb2a..4f306963 100644 --- a/libs/requests/sessions.py +++ b/libs/requests/sessions.py @@ -13,7 +13,7 @@ from collections import Mapping from datetime import datetime from .auth import _basic_auth_str -from .compat import cookielib, OrderedDict, urljoin, urlparse, builtin_str +from .compat import cookielib, OrderedDict, urljoin, urlparse from .cookies import ( cookiejar_from_dict, extract_cookies_to_jar, RequestsCookieJar, merge_cookies) from .models import Request, PreparedRequest, DEFAULT_REDIRECT_LIMIT @@ -21,6 +21,7 @@ from .hooks import default_hooks, dispatch_hook from .utils import to_key_val_list, default_headers, to_native_string from .exceptions import ( TooManyRedirects, InvalidSchema, ChunkedEncodingError, ContentDecodingError) +from .packages.urllib3._collections import RecentlyUsedContainer from .structures import CaseInsensitiveDict from .adapters import HTTPAdapter @@ -35,6 +36,8 @@ from .status_codes import codes # formerly defined here, reexposed here for backward compatibility from .models import REDIRECT_STATI +REDIRECT_CACHE_SIZE = 1000 + def merge_setting(request_setting, session_setting, dict_class=OrderedDict): """ @@ -128,14 +131,14 @@ class SessionRedirectMixin(object): # Facilitate relative 'location' headers, as allowed by RFC 7231. # (e.g. '/path/to/resource' instead of 'http://domain.tld/path/to/resource') # Compliant with RFC3986, we percent encode the url. - if not urlparse(url).netloc: + if not parsed.netloc: url = urljoin(resp.url, requote_uri(url)) else: url = requote_uri(url) prepared_request.url = to_native_string(url) - # cache the url - if resp.is_permanent_redirect: + # Cache the url, unless it redirects to itself. + if resp.is_permanent_redirect and req.url != prepared_request.url: self.redirect_cache[req.url] = prepared_request.url # http://tools.ietf.org/html/rfc7231#section-6.4.4 @@ -271,9 +274,10 @@ class Session(SessionRedirectMixin): """ __attrs__ = [ - 'headers', 'cookies', 'auth', 'timeout', 'proxies', 'hooks', - 'params', 'verify', 'cert', 'prefetch', 'adapters', 'stream', - 'trust_env', 'max_redirects', 'redirect_cache'] + 'headers', 'cookies', 'auth', 'proxies', 'hooks', 'params', 'verify', + 'cert', 'prefetch', 'adapters', 'stream', 'trust_env', + 'max_redirects', + ] def __init__(self): @@ -326,7 +330,8 @@ class Session(SessionRedirectMixin): self.mount('https://', HTTPAdapter()) self.mount('http://', HTTPAdapter()) - self.redirect_cache = {} + # Only store 1000 redirects to prevent using infinite memory + self.redirect_cache = RecentlyUsedContainer(REDIRECT_CACHE_SIZE) def __enter__(self): return self @@ -365,6 +370,7 @@ class Session(SessionRedirectMixin): url=request.url, files=request.files, data=request.data, + json=request.json, headers=merge_setting(request.headers, self.headers, dict_class=CaseInsensitiveDict), params=merge_setting(request.params, self.params), auth=merge_setting(auth, self.auth), @@ -386,7 +392,8 @@ class Session(SessionRedirectMixin): hooks=None, stream=None, verify=None, - cert=None): + cert=None, + json=None): """Constructs a :class:`Request `, prepares it and sends it. Returns :class:`Response ` object. @@ -396,17 +403,22 @@ class Session(SessionRedirectMixin): string for the :class:`Request`. :param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`. + :param json: (optional) json to send in the body of the + :class:`Request`. :param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`. :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`. - :param files: (optional) Dictionary of 'filename': file-like-objects + :param files: (optional) Dictionary of ``'filename': file-like-objects`` for multipart encoding upload. :param auth: (optional) Auth tuple or callable to enable Basic/Digest/Custom HTTP Auth. - :param timeout: (optional) Float describing the timeout of the - request in seconds. - :param allow_redirects: (optional) Boolean. Set to True by default. + :param timeout: (optional) How long to wait for the server to send + data before giving up, as a float, or a (`connect timeout, read + timeout `_) tuple. + :type timeout: float or tuple + :param allow_redirects: (optional) Set to True by default. + :type allow_redirects: bool :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy. :param stream: (optional) whether to immediately download the response @@ -417,7 +429,7 @@ class Session(SessionRedirectMixin): If Tuple, ('cert', 'key') pair. """ - method = builtin_str(method) + method = to_native_string(method) # Create the Request. req = Request( @@ -426,6 +438,7 @@ class Session(SessionRedirectMixin): headers = headers, files = files, data = data or {}, + json = json, params = params or {}, auth = auth, cookies = cookies, @@ -479,15 +492,16 @@ class Session(SessionRedirectMixin): kwargs.setdefault('allow_redirects', False) return self.request('HEAD', url, **kwargs) - def post(self, url, data=None, **kwargs): + def post(self, url, data=None, json=None, **kwargs): """Sends a POST request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. + :param json: (optional) json to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. """ - return self.request('POST', url, data=data, **kwargs) + return self.request('POST', url, data=data, json=json, **kwargs) def put(self, url, data=None, **kwargs): """Sends a PUT request. Returns :class:`Response` object. @@ -532,12 +546,13 @@ class Session(SessionRedirectMixin): if not isinstance(request, PreparedRequest): raise ValueError('You can only send PreparedRequests.') - redirect_count = 0 + checked_urls = set() while request.url in self.redirect_cache: - redirect_count += 1 - if redirect_count > self.max_redirects: - raise TooManyRedirects - request.url = self.redirect_cache.get(request.url) + checked_urls.add(request.url) + new_url = self.redirect_cache.get(request.url) + if new_url in checked_urls: + break + request.url = new_url # Set up variables needed for resolve_redirects and dispatching of hooks allow_redirects = kwargs.pop('allow_redirects', True) @@ -647,12 +662,19 @@ class Session(SessionRedirectMixin): self.adapters[key] = self.adapters.pop(key) def __getstate__(self): - return dict((attr, getattr(self, attr, None)) for attr in self.__attrs__) + state = dict((attr, getattr(self, attr, None)) for attr in self.__attrs__) + state['redirect_cache'] = dict(self.redirect_cache) + return state def __setstate__(self, state): + redirect_cache = state.pop('redirect_cache', {}) for attr, value in state.items(): setattr(self, attr, value) + self.redirect_cache = RecentlyUsedContainer(REDIRECT_CACHE_SIZE) + for redirect, to in redirect_cache.items(): + self.redirect_cache[redirect] = to + def session(): """Returns a :class:`Session` for context-management.""" diff --git a/libs/requests/utils.py b/libs/requests/utils.py index 2c6bb090..74679414 100644 --- a/libs/requests/utils.py +++ b/libs/requests/utils.py @@ -19,6 +19,7 @@ import re import sys import socket import struct +import warnings from . import __version__ from . import certs @@ -114,7 +115,7 @@ def get_netrc_auth(url): def guess_filename(obj): """Tries to guess the filename of the given object.""" name = getattr(obj, 'name', None) - if name and name[0] != '<' and name[-1] != '>': + if name and isinstance(name, builtin_str) and name[0] != '<' and name[-1] != '>': return os.path.basename(name) @@ -287,6 +288,11 @@ def get_encodings_from_content(content): :param content: bytestring to extract encodings from. """ + warnings.warn(( + 'In requests 3.0, get_encodings_from_content will be removed. For ' + 'more information, please see the discussion on issue #2266. (This' + ' warning should only appear once.)'), + DeprecationWarning) charset_re = re.compile(r']', flags=re.I) pragma_re = re.compile(r']', flags=re.I) @@ -351,12 +357,14 @@ def get_unicode_from_response(r): Tried: 1. charset from content-type - - 2. every encodings from ```` - - 3. fall back and replace all unicode characters + 2. fall back and replace all unicode characters """ + warnings.warn(( + 'In requests 3.0, get_unicode_from_response will be removed. For ' + 'more information, please see the discussion on issue #2266. (This' + ' warning should only appear once.)'), + DeprecationWarning) tried_encodings = [] @@ -555,7 +563,7 @@ def default_headers(): 'User-Agent': default_user_agent(), 'Accept-Encoding': ', '.join(('gzip', 'deflate')), 'Accept': '*/*', - 'Connection': 'keep-alive' + 'Connection': 'keep-alive', }) @@ -570,7 +578,7 @@ def parse_header_links(value): replace_chars = " '\"" - for val in value.split(","): + for val in re.split(", *<", value): try: url, params = val.split(";", 1) except ValueError: @@ -672,3 +680,18 @@ def to_native_string(string, encoding='ascii'): out = string.decode(encoding) return out + + +def urldefragauth(url): + """ + Given a url remove the fragment and the authentication part + """ + scheme, netloc, path, params, query, fragment = urlparse(url) + + # see func:`prepend_scheme_if_needed` + if not netloc: + netloc, path = path, netloc + + netloc = netloc.rsplit('@', 1)[-1] + + return urlunparse((scheme, netloc, path, params, query, '')) From 1510e37652a8eae74cd5346c918b3799bc87601b Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 11 Jan 2015 16:18:22 +0100 Subject: [PATCH 19/60] Update Tornado --- libs/tornado/httpclient.py | 13 ++++++++----- libs/tornado/httpserver.py | 28 ++++------------------------ libs/tornado/httputil.py | 16 +++++++++++++++- libs/tornado/iostream.py | 2 +- libs/tornado/netutil.py | 4 ++-- libs/tornado/options.py | 7 +++++++ libs/tornado/platform/kqueue.py | 3 +-- libs/tornado/platform/select.py | 2 +- libs/tornado/simple_httpclient.py | 8 ++------ libs/tornado/testing.py | 4 ++++ libs/tornado/web.py | 9 +++++---- libs/tornado/websocket.py | 2 +- libs/tornado/wsgi.py | 2 +- 13 files changed, 52 insertions(+), 48 deletions(-) diff --git a/libs/tornado/httpclient.py b/libs/tornado/httpclient.py index df429517..6ea872de 100755 --- a/libs/tornado/httpclient.py +++ b/libs/tornado/httpclient.py @@ -95,7 +95,8 @@ class HTTPClient(object): If it is a string, we construct an `HTTPRequest` using any additional kwargs: ``HTTPRequest(request, **kwargs)`` - If an error occurs during the fetch, we raise an `HTTPError`. + If an error occurs during the fetch, we raise an `HTTPError` unless + the ``raise_error`` keyword argument is set to False. """ response = self._io_loop.run_sync(functools.partial( self._async_client.fetch, request, **kwargs)) @@ -200,7 +201,7 @@ class AsyncHTTPClient(Configurable): raise RuntimeError("inconsistent AsyncHTTPClient cache") del self._instance_cache[self.io_loop] - def fetch(self, request, callback=None, **kwargs): + def fetch(self, request, callback=None, raise_error=True, **kwargs): """Executes a request, asynchronously returning an `HTTPResponse`. The request may be either a string URL or an `HTTPRequest` object. @@ -208,8 +209,10 @@ class AsyncHTTPClient(Configurable): kwargs: ``HTTPRequest(request, **kwargs)`` This method returns a `.Future` whose result is an - `HTTPResponse`. The ``Future`` will raise an `HTTPError` if - the request returned a non-200 response code. + `HTTPResponse`. By default, the ``Future`` will raise an `HTTPError` + if the request returned a non-200 response code. Instead, if + ``raise_error`` is set to False, the response will always be + returned regardless of the response code. If a ``callback`` is given, it will be invoked with the `HTTPResponse`. In the callback interface, `HTTPError` is not automatically raised. @@ -243,7 +246,7 @@ class AsyncHTTPClient(Configurable): future.add_done_callback(handle_future) def handle_response(response): - if response.error: + if raise_error and response.error: future.set_exception(response.error) else: future.set_result(response) diff --git a/libs/tornado/httpserver.py b/libs/tornado/httpserver.py index d4c990ca..47c74726 100755 --- a/libs/tornado/httpserver.py +++ b/libs/tornado/httpserver.py @@ -42,30 +42,10 @@ from tornado.tcpserver import TCPServer class HTTPServer(TCPServer, httputil.HTTPServerConnectionDelegate): r"""A non-blocking, single-threaded HTTP server. - A server is defined by either a request callback that takes a - `.HTTPServerRequest` as an argument or a `.HTTPServerConnectionDelegate` - instance. - - A simple example server that echoes back the URI you requested:: - - import tornado.httpserver - import tornado.ioloop - from tornado import httputil - - def handle_request(request): - message = "You requested %s\n" % request.uri - request.connection.write_headers( - httputil.ResponseStartLine('HTTP/1.1', 200, 'OK'), - httputil.HTTPHeaders({"Content-Length": str(len(message))})) - request.connection.write(message) - request.connection.finish() - - http_server = tornado.httpserver.HTTPServer(handle_request) - http_server.listen(8888) - tornado.ioloop.IOLoop.instance().start() - - Applications should use the methods of `.HTTPConnection` to write - their response. + A server is defined by a subclass of `.HTTPServerConnectionDelegate`, + or, for backwards compatibility, a callback that takes an + `.HTTPServerRequest` as an argument. The delegate is usually a + `tornado.web.Application`. `HTTPServer` supports keep-alive connections by default (automatically for HTTP/1.1, or for HTTP/1.0 when the client diff --git a/libs/tornado/httputil.py b/libs/tornado/httputil.py index f5c9c04f..88389fed 100755 --- a/libs/tornado/httputil.py +++ b/libs/tornado/httputil.py @@ -331,7 +331,7 @@ class HTTPServerRequest(object): self.uri = uri self.version = version self.headers = headers or HTTPHeaders() - self.body = body or "" + self.body = body or b"" # set remote IP and protocol context = getattr(connection, 'context', None) @@ -873,3 +873,17 @@ def _encode_header(key, pdict): def doctests(): import doctest return doctest.DocTestSuite() + +def split_host_and_port(netloc): + """Returns ``(host, port)`` tuple from ``netloc``. + + Returned ``port`` will be ``None`` if not present. + """ + match = re.match(r'^(.+):(\d+)$', netloc) + if match: + host = match.group(1) + port = int(match.group(2)) + else: + host = netloc + port = None + return (host, port) diff --git a/libs/tornado/iostream.py b/libs/tornado/iostream.py index eced6d64..2d5df992 100755 --- a/libs/tornado/iostream.py +++ b/libs/tornado/iostream.py @@ -331,7 +331,7 @@ class BaseIOStream(object): if data: if (self.max_write_buffer_size is not None and self._write_buffer_size + len(data) > self.max_write_buffer_size): - raise StreamBufferFullError("Reached maximum read buffer size") + raise StreamBufferFullError("Reached maximum write buffer size") # Break up large contiguous strings before inserting them in the # write buffer, so we don't have to recopy the entire thing # as we slice off pieces to send to the socket. diff --git a/libs/tornado/netutil.py b/libs/tornado/netutil.py index f147c974..e85f62b7 100755 --- a/libs/tornado/netutil.py +++ b/libs/tornado/netutil.py @@ -20,7 +20,7 @@ from __future__ import absolute_import, division, print_function, with_statement import errno import os -import platform +import sys import socket import stat @@ -105,7 +105,7 @@ def bind_sockets(port, address=None, family=socket.AF_UNSPEC, for res in set(socket.getaddrinfo(address, port, family, socket.SOCK_STREAM, 0, flags)): af, socktype, proto, canonname, sockaddr = res - if (platform.system() == 'Darwin' and address == 'localhost' and + if (sys.platform == 'darwin' and address == 'localhost' and af == socket.AF_INET6 and sockaddr[3] != 0): # Mac OS X includes a link-local address fe80::1%lo0 in the # getaddrinfo results for 'localhost'. However, the firewall diff --git a/libs/tornado/options.py b/libs/tornado/options.py index 5e23e291..c855407c 100755 --- a/libs/tornado/options.py +++ b/libs/tornado/options.py @@ -204,6 +204,13 @@ class OptionParser(object): (name, self._options[name].file_name)) frame = sys._getframe(0) options_file = frame.f_code.co_filename + + # Can be called directly, or through top level define() fn, in which + # case, step up above that frame to look for real caller. + if (frame.f_back.f_code.co_filename == options_file and + frame.f_back.f_code.co_name == 'define'): + frame = frame.f_back + file_name = frame.f_back.f_code.co_filename if file_name == options_file: file_name = "" diff --git a/libs/tornado/platform/kqueue.py b/libs/tornado/platform/kqueue.py index de8c046d..f8f3e4a6 100755 --- a/libs/tornado/platform/kqueue.py +++ b/libs/tornado/platform/kqueue.py @@ -54,8 +54,7 @@ class _KQueue(object): if events & IOLoop.WRITE: kevents.append(select.kevent( fd, filter=select.KQ_FILTER_WRITE, flags=flags)) - if events & IOLoop.READ or not kevents: - # Always read when there is not a write + if events & IOLoop.READ: kevents.append(select.kevent( fd, filter=select.KQ_FILTER_READ, flags=flags)) # Even though control() takes a list, it seems to return EINVAL diff --git a/libs/tornado/platform/select.py b/libs/tornado/platform/select.py index 9a879562..1e126554 100755 --- a/libs/tornado/platform/select.py +++ b/libs/tornado/platform/select.py @@ -47,7 +47,7 @@ class _Select(object): # Closed connections are reported as errors by epoll and kqueue, # but as zero-byte reads by select, so when errors are requested # we need to listen for both read and error. - self.read_fds.add(fd) + #self.read_fds.add(fd) def modify(self, fd, events): self.unregister(fd) diff --git a/libs/tornado/simple_httpclient.py b/libs/tornado/simple_httpclient.py index e60c434f..7c915e90 100755 --- a/libs/tornado/simple_httpclient.py +++ b/libs/tornado/simple_httpclient.py @@ -193,12 +193,8 @@ class _HTTPConnection(httputil.HTTPMessageDelegate): netloc = self.parsed.netloc if "@" in netloc: userpass, _, netloc = netloc.rpartition("@") - match = re.match(r'^(.+):(\d+)$', netloc) - if match: - host = match.group(1) - port = int(match.group(2)) - else: - host = netloc + host, port = httputil.split_host_and_port(netloc) + if port is None: port = 443 if self.parsed.scheme == "https" else 80 if re.match(r'^\[.*\]$', host): # raw ipv6 addresses in urls are enclosed in brackets diff --git a/libs/tornado/testing.py b/libs/tornado/testing.py index 4d85abe9..4511863b 100755 --- a/libs/tornado/testing.py +++ b/libs/tornado/testing.py @@ -19,6 +19,7 @@ try: from tornado.simple_httpclient import SimpleAsyncHTTPClient from tornado.ioloop import IOLoop, TimeoutError from tornado import netutil + from tornado.process import Subprocess except ImportError: # These modules are not importable on app engine. Parts of this module # won't work, but e.g. LogTrapTestCase and main() will. @@ -28,6 +29,7 @@ except ImportError: IOLoop = None netutil = None SimpleAsyncHTTPClient = None + Subprocess = None from tornado.log import gen_log, app_log from tornado.stack_context import ExceptionStackContext from tornado.util import raise_exc_info, basestring_type @@ -214,6 +216,8 @@ class AsyncTestCase(unittest.TestCase): self.io_loop.make_current() def tearDown(self): + # Clean up Subprocess, so it can be used again with a new ioloop. + Subprocess.uninitialize() self.io_loop.clear_current() if (not IOLoop.initialized() or self.io_loop is not IOLoop.instance()): diff --git a/libs/tornado/web.py b/libs/tornado/web.py index a038265f..2d1dac0f 100755 --- a/libs/tornado/web.py +++ b/libs/tornado/web.py @@ -85,6 +85,7 @@ from tornado import stack_context from tornado import template from tornado.escape import utf8, _unicode from tornado.util import import_object, ObjectDict, raise_exc_info, unicode_type, _websocket_mask +from tornado.httputil import split_host_and_port try: @@ -1477,7 +1478,7 @@ def asynchronous(method): with stack_context.ExceptionStackContext( self._stack_context_handle_exception): result = method(self, *args, **kwargs) - if isinstance(result, Future): + if is_future(result): # If @asynchronous is used with @gen.coroutine, (but # not @gen.engine), we can automatically finish the # request when the future resolves. Additionally, @@ -1518,7 +1519,7 @@ def stream_request_body(cls): the entire body has been read. There is a subtle interaction between ``data_received`` and asynchronous - ``prepare``: The first call to ``data_recieved`` may occur at any point + ``prepare``: The first call to ``data_received`` may occur at any point after the call to ``prepare`` has returned *or yielded*. """ if not issubclass(cls, RequestHandler): @@ -1729,7 +1730,7 @@ class Application(httputil.HTTPServerConnectionDelegate): self.transforms.append(transform_class) def _get_host_handlers(self, request): - host = request.host.lower().split(':')[0] + host = split_host_and_port(request.host.lower())[0] matches = [] for pattern, handlers in self.handlers: if pattern.match(host): @@ -1845,7 +1846,7 @@ class _RequestDispatcher(httputil.HTTPMessageDelegate): handlers = app._get_host_handlers(self.request) if not handlers: self.handler_class = RedirectHandler - self.handler_kwargs = dict(url="http://" + app.default_host + "/") + self.handler_kwargs = dict(url="%s://%s/" % (self.request.protocol, app.default_host)) return for spec in handlers: match = spec.regex.match(self.request.path) diff --git a/libs/tornado/websocket.py b/libs/tornado/websocket.py index d960b0e4..5c762adb 100755 --- a/libs/tornado/websocket.py +++ b/libs/tornado/websocket.py @@ -229,7 +229,7 @@ class WebSocketHandler(tornado.web.RequestHandler): """ return None - def open(self): + def open(self, *args, **kwargs): """Invoked when a new WebSocket is opened. The arguments to `open` are extracted from the `tornado.web.URLSpec` diff --git a/libs/tornado/wsgi.py b/libs/tornado/wsgi.py index f3aa6650..e7e07fbc 100755 --- a/libs/tornado/wsgi.py +++ b/libs/tornado/wsgi.py @@ -207,7 +207,7 @@ class WSGIAdapter(object): body = environ["wsgi.input"].read( int(headers["Content-Length"])) else: - body = "" + body = b"" protocol = environ["wsgi.url_scheme"] remote_ip = environ.get("REMOTE_ADDR", "") if environ.get("HTTP_HOST"): From ee8406e026aa9b6751277373752f388af764aa2b Mon Sep 17 00:00:00 2001 From: Andrew Dumaresq Date: Sun, 11 Jan 2015 11:45:29 -0500 Subject: [PATCH 20/60] Minor text change --- couchpotato/core/downloaders/putio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/downloaders/putio/__init__.py b/couchpotato/core/downloaders/putio/__init__.py index 60ccad58..48794d9d 100644 --- a/couchpotato/core/downloaders/putio/__init__.py +++ b/couchpotato/core/downloaders/putio/__init__.py @@ -30,7 +30,7 @@ config = [{ }, { 'name': 'folder', - 'description': ('The folder on putio where you want the upload to go','Must be a folder in the root directory'), + 'description': ('The folder on putio where you want the upload to go','Will find the first first folder that matches this name'), 'default': 0, }, { From 20e12836270af7cb523883b17b0271006fdd8573 Mon Sep 17 00:00:00 2001 From: Andrew Dumaresq Date: Sun, 11 Jan 2015 11:57:14 -0500 Subject: [PATCH 21/60] better way to find the folder --- couchpotato/core/downloaders/putio/main.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/couchpotato/core/downloaders/putio/main.py b/couchpotato/core/downloaders/putio/main.py index ce58ff7c..478d15c9 100644 --- a/couchpotato/core/downloaders/putio/main.py +++ b/couchpotato/core/downloaders/putio/main.py @@ -28,6 +28,19 @@ class PutIO(DownloaderBase): return super(PutIO, self).__init__() + # This is a recusive function to check for the folders + def recursionFolder(self, client, folder, tfolder): + files = client.File.list(folder) + for f in files: + if f.name == tfolder and f.content_type == "application/x-directory": + return f.id + elif f.content_type == "application/x-directory": + result = self.recursionFolder(client, f.id, tfolder) + if result != 0: + return result + return 0 + + # This will check the root for the folder, and kick of recusively checking sub folder def convertFolder(self, client, folder): if folder == 0: return 0 @@ -36,6 +49,10 @@ class PutIO(DownloaderBase): for f in files: if f.name == folder and f.content_type == "application/x-directory": return f.id + elif f.content_type == "application/x-directory": + result = self.recursionFolder(client, f.id, folder) + if result != 0: + return result #If we get through the whole list and don't get a match we will use the root return 0 From 89836be1d1c21e3bd71790fb5afe78e1a9d164c4 Mon Sep 17 00:00:00 2001 From: David Stark Date: Mon, 12 Jan 2015 17:37:26 +0100 Subject: [PATCH 22/60] added touch and chown to the $PID_FILE --- init/ubuntu | 2 ++ 1 file changed, 2 insertions(+) diff --git a/init/ubuntu b/init/ubuntu index a21e2d34..cbe20e08 100755 --- a/init/ubuntu +++ b/init/ubuntu @@ -95,6 +95,8 @@ fi case "$1" in start) + touch $PID_FILE + chown $RUN_AS $PID_FILE echo "Starting $DESC" start-stop-daemon -d $APP_PATH -c $RUN_AS $EXTRA_SSD_OPTS --start --pidfile $PID_FILE --exec $DAEMON -- $DAEMON_OPTS ;; From 6aca799bbb9aa8ddc429e29e2936e834caa778e2 Mon Sep 17 00:00:00 2001 From: Ruud Date: Wed, 14 Jan 2015 16:55:30 +0100 Subject: [PATCH 23/60] Newznab: use guid for detail url --- couchpotato/core/media/_base/providers/nzb/newznab.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/couchpotato/core/media/_base/providers/nzb/newznab.py b/couchpotato/core/media/_base/providers/nzb/newznab.py index 21ce94b3..68acd6b9 100644 --- a/couchpotato/core/media/_base/providers/nzb/newznab.py +++ b/couchpotato/core/media/_base/providers/nzb/newznab.py @@ -68,7 +68,8 @@ class Base(NZBProvider, RSS): if not date: date = self.getTextElement(nzb, 'pubDate') - nzb_id = self.getTextElement(nzb, 'guid').split('/')[-1:].pop() + detail_url = self.getTextElement(nzb, 'guid') + nzb_id = detail_url.split('/')[-1:].pop() name = self.getTextElement(nzb, 'title') if not name: From ab61961a64f80b7ff5a1cce477d74b663bfaf7cc Mon Sep 17 00:00:00 2001 From: Ruud Date: Wed, 14 Jan 2015 16:59:29 +0100 Subject: [PATCH 24/60] Use detail url --- couchpotato/core/media/_base/providers/nzb/newznab.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/couchpotato/core/media/_base/providers/nzb/newznab.py b/couchpotato/core/media/_base/providers/nzb/newznab.py index 68acd6b9..9a12ff91 100644 --- a/couchpotato/core/media/_base/providers/nzb/newznab.py +++ b/couchpotato/core/media/_base/providers/nzb/newznab.py @@ -20,7 +20,6 @@ log = CPLog(__name__) class Base(NZBProvider, RSS): urls = { - 'detail': 'details/%s', 'download': 't=get&id=%s' } @@ -104,7 +103,7 @@ class Base(NZBProvider, RSS): 'age': self.calculateAge(int(time.mktime(parse(date).timetuple()))), 'size': int(self.getElement(nzb, 'enclosure').attrib['length']) / 1024 / 1024, 'url': ((self.getUrl(host['host']) + self.urls['download']) % tryUrlencode(nzb_id)) + self.getApiExt(host), - 'detail_url': (cleanHost(host['host']) + self.urls['detail']) % tryUrlencode(nzb_id), + 'detail_url': detail_url, 'content': self.getTextElement(nzb, 'description'), 'description': description, 'score': host['extra_score'], From 770c2be14c4594a1acdae5bb5ecd5bd378b31e2b Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 17 Jan 2015 13:04:47 +0100 Subject: [PATCH 25/60] Create detail url if permalink is false --- couchpotato/core/media/_base/providers/nzb/newznab.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/couchpotato/core/media/_base/providers/nzb/newznab.py b/couchpotato/core/media/_base/providers/nzb/newznab.py index 9a12ff91..b9aad556 100644 --- a/couchpotato/core/media/_base/providers/nzb/newznab.py +++ b/couchpotato/core/media/_base/providers/nzb/newznab.py @@ -20,6 +20,7 @@ log = CPLog(__name__) class Base(NZBProvider, RSS): urls = { + 'detail': 'details/%s', 'download': 't=get&id=%s' } @@ -67,9 +68,12 @@ class Base(NZBProvider, RSS): if not date: date = self.getTextElement(nzb, 'pubDate') + name = self.getTextElement(nzb, 'title') detail_url = self.getTextElement(nzb, 'guid') nzb_id = detail_url.split('/')[-1:].pop() - name = self.getTextElement(nzb, 'title') + + if '://' not in detail_url: + detail_url = (cleanHost(host['host']) + self.urls['detail']) % tryUrlencode(nzb_id) if not name: continue From e52f50b204902551043bf3aedc6f6c3f29326835 Mon Sep 17 00:00:00 2001 From: coolius Date: Mon, 19 Jan 2015 17:17:31 +0000 Subject: [PATCH 26/60] Update torrentday.py Updated torrentday url to blockade-free torrentday.eu --- .../media/_base/providers/torrent/torrentday.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/couchpotato/core/media/_base/providers/torrent/torrentday.py b/couchpotato/core/media/_base/providers/torrent/torrentday.py index a3e9b78c..496debe5 100644 --- a/couchpotato/core/media/_base/providers/torrent/torrentday.py +++ b/couchpotato/core/media/_base/providers/torrent/torrentday.py @@ -8,12 +8,12 @@ log = CPLog(__name__) class Base(TorrentProvider): urls = { - 'test': 'http://www.td.af/', - 'login': 'http://www.td.af/torrents/', - 'login_check': 'http://www.torrentday.com/userdetails.php', - 'detail': 'http://www.td.af/details.php?id=%s', - 'search': 'http://www.td.af/V3/API/API.php', - 'download': 'http://www.td.af/download.php/%s/%s', + 'test': 'https://torrentday.eu/', + 'login': 'https://torrentday.eu/torrents/', + 'login_check': 'https://torrentday.eu/userdetails.php', + 'detail': 'https://torrentday.eu/details.php?id=%s', + 'search': 'https://torrentday.eu/V3/API/API.php', + 'download': 'https://torrentday.eu/download.php/%s/%s', } http_time_between_calls = 1 # Seconds @@ -68,7 +68,7 @@ config = [{ 'tab': 'searcher', 'list': 'torrent_providers', 'name': 'TorrentDay', - 'description': 'TorrentDay', + 'description': 'TorrentDay', 'wizard': True, 'icon': 'iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAC5ElEQVQ4y12TXUgUURTH//fO7Di7foeQJH6gEEEIZZllVohfSG/6UA+RSFAQQj74VA8+Bj30lmAlRVSEvZRfhNhaka5ZUG1paKaW39tq5O6Ou+PM3M4o6m6X+XPPzD3zm/+dcy574r515WfIW8CZBM4YAA5Gc/aQC3yd7oXYEONcsISE5dTDh91HS0t7FEWhBUAeN9ynV/d9qJAgE4AECURAcVsGlCCnly26LMA0IQwTa52dje3d3e3hcPi8qqrrMjcVYI3EHCQZlkFOHBwR2QHh2ASAAIJxWGAQEDxjePhs3527XjJwnb37OHBq0T+Tyyjh+9KnEzNJ7nouc1Q/3A3HGsOvnJy+PSUlj81w2Lny9WuJ6+3AmTjD4HOcrdR2dWXLRQePvyaSLfQOPMPC8mC9iHCsOxSyzJCelzdSXlNzD5ujpb25Wbfc/XXJemTXF4+nnCNq+AMLe50uFfEJTiw4GXSFtiHL0SnIq66+p0kSArqO+eH3RdsAv9+f5vW7L7GICq6rmM8XBCAXlBw90rOyxibn5yzfkg/L09M52/jxqdESaIrBXHYZZbB1GX8cEpySxKIB8S5XcOnvqpli1zuwmrTtoLjw5LOK/eeuWsE4JH5IRPaPZKiKigmPp+5pa+u1aEjIMhEgrRkmi9mgxGUhM7LNJSzOzsE3+cOeExovXOjdytE0LV4zqNZUtV0uZzAGoGkhDH/2YHZiErmv4uyWQnZZWc+hoqL3WzlTExN5hhA8IEwkZWZOxwB++30YG/9GkYCPvqAaHAW5uWPROW86OmqCprUR7z1yZDAGQNuCvkoB/baIKUBWMTYymv+gra3eJNvjXu+B562tFyXqTJ6YuHK8rKwvBmC3vR7cOCPQLWFz8LnfXWUrJo9U19BwMyUlJRjTSMJ2ENxUiGxq9KXQfwqYlnWstvbR5aamG9g0uzM8Q4OFt++3NNixQ2NgYmeN03FOTUv7XVpV9aKisvLl1vN/WVhNc/Fi1NEAAAAASUVORK5CYII=', 'options': [ From cdb9cfe75652cca2bdc28d7d69218f800b4942fd Mon Sep 17 00:00:00 2001 From: coolius Date: Mon, 19 Jan 2015 17:18:56 +0000 Subject: [PATCH 27/60] Update iptorrents.py Updated iptorrents url to blockade-free iptorrents.eu --- .../core/media/_base/providers/torrent/iptorrents.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/couchpotato/core/media/_base/providers/torrent/iptorrents.py b/couchpotato/core/media/_base/providers/torrent/iptorrents.py index 0915ca31..6d289c34 100644 --- a/couchpotato/core/media/_base/providers/torrent/iptorrents.py +++ b/couchpotato/core/media/_base/providers/torrent/iptorrents.py @@ -14,11 +14,11 @@ log = CPLog(__name__) class Base(TorrentProvider): urls = { - 'test': 'https://www.iptorrents.com/', - 'base_url': 'https://www.iptorrents.com', - 'login': 'https://www.iptorrents.com/torrents/', - 'login_check': 'https://www.iptorrents.com/inbox.php', - 'search': 'https://www.iptorrents.com/torrents/?%s%%s&q=%s&qf=ti&p=%%d', + 'test': 'https://www.iptorrents.eu/', + 'base_url': 'https://www.iptorrents.eu', + 'login': 'https://www.iptorrents.eu/torrents/', + 'login_check': 'https://www.iptorrents.eu/inbox.php', + 'search': 'https://www.iptorrents.eu/torrents/?%s%%s&q=%s&qf=ti&p=%%d', } http_time_between_calls = 1 # Seconds @@ -120,7 +120,7 @@ config = [{ 'tab': 'searcher', 'list': 'torrent_providers', 'name': 'IPTorrents', - 'description': 'IPTorrents', + 'description': 'IPTorrents', 'wizard': True, 'icon': 'iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAABRklEQVR42qWQO0vDUBiG8zeKY3EqQUtNO7g0J6ZJ1+ifKIIFQXAqDYKCyaaYxM3udrZLHdRFhXrZ6liCW6mubfk874EESgqaeOCF7/Y8hEh41aq6yZi2nyZgBGya9XKtZs4No05pAkZV2YbEmyMMsoSxLQeC46wCTdPPY4HruPQyGIhF97qLWsS78Miydn4XdK46NJ9OsQAYBzMIMf8MQ9wtCnTdWCaIDx/u7uljOIQEe0hiIWPamSTLay3+RxOCSPI9+RJAo7Er9r2bnqjBFAqyK+VyK4f5/Cr5ni8OFKVCz49PFI5GdNvvU7ttE1M1zMU+8AMqFksEhrMnQsBDzqmDAwzx2ehRLwT7yyCI+vSC99c3mozH1NxrJgWWtR1BOECfEJSVCm6WCzJGCA7+IWhBsM4zywDPwEp4vCjx2DzBH2ODAfsDb33Ps6dQwJgAAAAASUVORK5CYII=', 'options': [ From c9b4c8167f6f76bd121091a2cac9d671cc7481f8 Mon Sep 17 00:00:00 2001 From: Ruud Date: Wed, 28 Jan 2015 11:35:26 +0100 Subject: [PATCH 28/60] Actual include host in log --- couchpotato/core/plugins/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/plugins/base.py b/couchpotato/core/plugins/base.py index c02e8f75..e4b27c9b 100644 --- a/couchpotato/core/plugins/base.py +++ b/couchpotato/core/plugins/base.py @@ -206,7 +206,7 @@ class Plugin(object): if self.http_failed_disabled[host] > (time.time() - 900): log.info2('Disabled calls to %s for 15 minutes because so many failed requests.', host) if not show_error: - raise Exception('Disabled calls to %s for 15 minutes because so many failed requests') + raise Exception('Disabled calls to %s for 15 minutes because so many failed requests' % host) else: return '' else: From b00e69e222edd2b33e7daea8c203a7bcfb34dd45 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 31 Jan 2015 10:32:15 +0100 Subject: [PATCH 29/60] TorrentBytes cut of longer titles fix #4590 --- couchpotato/core/media/_base/providers/torrent/torrentbytes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/media/_base/providers/torrent/torrentbytes.py b/couchpotato/core/media/_base/providers/torrent/torrentbytes.py index fadd2ea0..32221d83 100644 --- a/couchpotato/core/media/_base/providers/torrent/torrentbytes.py +++ b/couchpotato/core/media/_base/providers/torrent/torrentbytes.py @@ -56,7 +56,7 @@ class Base(TorrentProvider): full_id = link['href'].replace('details.php?id=', '') torrent_id = full_id[:6] - name = toUnicode(link.contents[0].encode('ISO-8859-1')).strip() + name = toUnicode(link.get('title', link.contents[0]).encode('ISO-8859-1')).strip() results.append({ 'id': torrent_id, From e8a3645bc68d6c2d22c8112ea62668157ff0eb10 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 1 Feb 2015 12:18:31 +0100 Subject: [PATCH 30/60] Log failed folder getting --- couchpotato/core/plugins/browser.py | 1 + 1 file changed, 1 insertion(+) diff --git a/couchpotato/core/plugins/browser.py b/couchpotato/core/plugins/browser.py index 632375df..660070a2 100644 --- a/couchpotato/core/plugins/browser.py +++ b/couchpotato/core/plugins/browser.py @@ -87,6 +87,7 @@ class FileBrowser(Plugin): try: dirs = self.getDirectories(path = path, show_hidden = show_hidden) except: + log.error('Failed getting directory "%s" : %s', (path, traceback.format_exc())) dirs = [] parent = os.path.dirname(path.rstrip(os.path.sep)) From fb8a66d2078d5ead721b3f857e8aeb47c4d1181d Mon Sep 17 00:00:00 2001 From: maikhorma Date: Sun, 1 Feb 2015 14:43:16 -0500 Subject: [PATCH 31/60] Shortcut to address #2782 Until there is a more elegant solution to avoid unwanted white space trimming, this will let users disable that feature if it is not something they need. --- couchpotato/core/plugins/renamer.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/couchpotato/core/plugins/renamer.py b/couchpotato/core/plugins/renamer.py index d0720d0b..df25fcaa 100755 --- a/couchpotato/core/plugins/renamer.py +++ b/couchpotato/core/plugins/renamer.py @@ -885,7 +885,9 @@ Remove it if you want it to be renamed (again, or at least let it try again) #If information is not available, we don't want the tag in the filename replaced = replaced.replace('<' + x + '>', '') - replaced = self.replaceDoubles(replaced.lstrip('. ')) + if self.conf('replace_doubles'): + replaced = self.replaceDoubles(replaced.lstrip('. ')) + for x, r in replacements.items(): if x in ['thename', 'namethe']: replaced = replaced.replace(six.u('<%s>') % toUnicode(x), toUnicode(r)) @@ -1342,6 +1344,14 @@ config = [{ 'type': 'choice', 'options': rename_options }, + { + 'advanced': True, + 'name': 'replace_doubles', + 'type': 'bool', + 'label': 'Consider Missing Data', + 'description': 'Attempt to clean up double separaters due to missing data for fields', + 'default': True + }, { 'name': 'unrar', 'type': 'bool', From cf83f99be07a69ca275ee318d6879c409e18c5ac Mon Sep 17 00:00:00 2001 From: maikhorma Date: Sun, 1 Feb 2015 15:28:05 -0500 Subject: [PATCH 32/60] Updated UI Tried to make it a bit cleaner. --- couchpotato/core/plugins/renamer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/couchpotato/core/plugins/renamer.py b/couchpotato/core/plugins/renamer.py index df25fcaa..60b4f8b6 100755 --- a/couchpotato/core/plugins/renamer.py +++ b/couchpotato/core/plugins/renamer.py @@ -1348,8 +1348,8 @@ config = [{ 'advanced': True, 'name': 'replace_doubles', 'type': 'bool', - 'label': 'Consider Missing Data', - 'description': 'Attempt to clean up double separaters due to missing data for fields', + 'label': 'Clean Name', + 'description': ('Attempt to clean up double separaters due to missing data for fields.','Sometimes this eliminates wanted white space (see #2782).'), 'default': True }, { From 7b9043c16bb55a93898dae5f4b0d6754a78d3c1e Mon Sep 17 00:00:00 2001 From: sammy2142 Date: Tue, 10 Feb 2015 11:11:30 +0000 Subject: [PATCH 33/60] Update kickass url from kickass.so to kickass.to Kickass has reverted back to the .to domain as the .so domain was seized: http://torrentfreak.com/kickasstorrents-taken-domain-name-seizure-150209/ --- .../core/media/_base/providers/torrent/kickasstorrents.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/media/_base/providers/torrent/kickasstorrents.py b/couchpotato/core/media/_base/providers/torrent/kickasstorrents.py index fb58814d..d6e3ee72 100644 --- a/couchpotato/core/media/_base/providers/torrent/kickasstorrents.py +++ b/couchpotato/core/media/_base/providers/torrent/kickasstorrents.py @@ -30,7 +30,7 @@ class Base(TorrentMagnetProvider): cat_backup_id = None proxy_list = [ - 'https://kickass.so', + 'https://kickass.to', 'http://kickass.pw', 'http://kickassto.come.in', 'http://katproxy.ws', From 7a616a81f7bcb5f7f9a04c603493d80dd10ebb50 Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 10 Feb 2015 12:52:05 +0100 Subject: [PATCH 34/60] Remove www from iptorrents --- .../core/media/_base/providers/torrent/iptorrents.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/couchpotato/core/media/_base/providers/torrent/iptorrents.py b/couchpotato/core/media/_base/providers/torrent/iptorrents.py index 6d289c34..61ced9c9 100644 --- a/couchpotato/core/media/_base/providers/torrent/iptorrents.py +++ b/couchpotato/core/media/_base/providers/torrent/iptorrents.py @@ -14,11 +14,11 @@ log = CPLog(__name__) class Base(TorrentProvider): urls = { - 'test': 'https://www.iptorrents.eu/', - 'base_url': 'https://www.iptorrents.eu', - 'login': 'https://www.iptorrents.eu/torrents/', - 'login_check': 'https://www.iptorrents.eu/inbox.php', - 'search': 'https://www.iptorrents.eu/torrents/?%s%%s&q=%s&qf=ti&p=%%d', + 'test': 'https://iptorrents.eu/', + 'base_url': 'https://iptorrents.eu', + 'login': 'https://iptorrents.eu/torrents/', + 'login_check': 'https://iptorrents.eu/inbox.php', + 'search': 'https://iptorrents.eu/torrents/?%s%%s&q=%s&qf=ti&p=%%d', } http_time_between_calls = 1 # Seconds @@ -120,7 +120,7 @@ config = [{ 'tab': 'searcher', 'list': 'torrent_providers', 'name': 'IPTorrents', - 'description': 'IPTorrents', + 'description': 'IPTorrents', 'wizard': True, 'icon': 'iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAABRklEQVR42qWQO0vDUBiG8zeKY3EqQUtNO7g0J6ZJ1+ifKIIFQXAqDYKCyaaYxM3udrZLHdRFhXrZ6liCW6mubfk874EESgqaeOCF7/Y8hEh41aq6yZi2nyZgBGya9XKtZs4No05pAkZV2YbEmyMMsoSxLQeC46wCTdPPY4HruPQyGIhF97qLWsS78Miydn4XdK46NJ9OsQAYBzMIMf8MQ9wtCnTdWCaIDx/u7uljOIQEe0hiIWPamSTLay3+RxOCSPI9+RJAo7Er9r2bnqjBFAqyK+VyK4f5/Cr5ni8OFKVCz49PFI5GdNvvU7ttE1M1zMU+8AMqFksEhrMnQsBDzqmDAwzx2ehRLwT7yyCI+vSC99c3mozH1NxrJgWWtR1BOECfEJSVCm6WCzJGCA7+IWhBsM4zywDPwEp4vCjx2DzBH2ODAfsDb33Ps6dQwJgAAAAASUVORK5CYII=', 'options': [ From d5622b7cba18928acf01430ecd0033c258a95adb Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 10 Feb 2015 13:01:19 +0100 Subject: [PATCH 35/60] Remove www from torrentday domain --- couchpotato/core/media/_base/providers/torrent/torrentday.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/media/_base/providers/torrent/torrentday.py b/couchpotato/core/media/_base/providers/torrent/torrentday.py index 496debe5..57224ffe 100644 --- a/couchpotato/core/media/_base/providers/torrent/torrentday.py +++ b/couchpotato/core/media/_base/providers/torrent/torrentday.py @@ -68,7 +68,7 @@ config = [{ 'tab': 'searcher', 'list': 'torrent_providers', 'name': 'TorrentDay', - 'description': 'TorrentDay', + 'description': 'TorrentDay', 'wizard': True, 'icon': 'iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAC5ElEQVQ4y12TXUgUURTH//fO7Di7foeQJH6gEEEIZZllVohfSG/6UA+RSFAQQj74VA8+Bj30lmAlRVSEvZRfhNhaka5ZUG1paKaW39tq5O6Ou+PM3M4o6m6X+XPPzD3zm/+dcy574r515WfIW8CZBM4YAA5Gc/aQC3yd7oXYEONcsISE5dTDh91HS0t7FEWhBUAeN9ynV/d9qJAgE4AECURAcVsGlCCnly26LMA0IQwTa52dje3d3e3hcPi8qqrrMjcVYI3EHCQZlkFOHBwR2QHh2ASAAIJxWGAQEDxjePhs3527XjJwnb37OHBq0T+Tyyjh+9KnEzNJ7nouc1Q/3A3HGsOvnJy+PSUlj81w2Lny9WuJ6+3AmTjD4HOcrdR2dWXLRQePvyaSLfQOPMPC8mC9iHCsOxSyzJCelzdSXlNzD5ujpb25Wbfc/XXJemTXF4+nnCNq+AMLe50uFfEJTiw4GXSFtiHL0SnIq66+p0kSArqO+eH3RdsAv9+f5vW7L7GICq6rmM8XBCAXlBw90rOyxibn5yzfkg/L09M52/jxqdESaIrBXHYZZbB1GX8cEpySxKIB8S5XcOnvqpli1zuwmrTtoLjw5LOK/eeuWsE4JH5IRPaPZKiKigmPp+5pa+u1aEjIMhEgrRkmi9mgxGUhM7LNJSzOzsE3+cOeExovXOjdytE0LV4zqNZUtV0uZzAGoGkhDH/2YHZiErmv4uyWQnZZWc+hoqL3WzlTExN5hhA8IEwkZWZOxwB++30YG/9GkYCPvqAaHAW5uWPROW86OmqCprUR7z1yZDAGQNuCvkoB/baIKUBWMTYymv+gra3eJNvjXu+B562tFyXqTJ6YuHK8rKwvBmC3vR7cOCPQLWFz8LnfXWUrJo9U19BwMyUlJRjTSMJ2ENxUiGxq9KXQfwqYlnWstvbR5aamG9g0uzM8Q4OFt++3NNixQ2NgYmeN03FOTUv7XVpV9aKisvLl1vN/WVhNc/Fi1NEAAAAASUVORK5CYII=', 'options': [ From c1266a36e4c526017958b76c41973707f54c1b65 Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 10 Feb 2015 13:15:08 +0100 Subject: [PATCH 36/60] Re-use resursion code --- couchpotato/core/downloaders/putio/main.py | 26 ++++++++-------------- 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/couchpotato/core/downloaders/putio/main.py b/couchpotato/core/downloaders/putio/main.py index 478d15c9..c5685821 100644 --- a/couchpotato/core/downloaders/putio/main.py +++ b/couchpotato/core/downloaders/putio/main.py @@ -29,15 +29,16 @@ class PutIO(DownloaderBase): return super(PutIO, self).__init__() # This is a recusive function to check for the folders - def recursionFolder(self, client, folder, tfolder): + def recursionFolder(self, client, folder = 0, tfolder = ''): files = client.File.list(folder) for f in files: - if f.name == tfolder and f.content_type == "application/x-directory": - return f.id - elif f.content_type == "application/x-directory": - result = self.recursionFolder(client, f.id, tfolder) - if result != 0: - return result + if f.content_type == 'application/x-directory': + if f.name == tfolder: + return f.id + else: + result = self.recursionFolder(client, f.id, tfolder) + if result != 0: + return result return 0 # This will check the root for the folder, and kick of recusively checking sub folder @@ -45,16 +46,7 @@ class PutIO(DownloaderBase): if folder == 0: return 0 else: - files = client.File.list() - for f in files: - if f.name == folder and f.content_type == "application/x-directory": - return f.id - elif f.content_type == "application/x-directory": - result = self.recursionFolder(client, f.id, folder) - if result != 0: - return result - #If we get through the whole list and don't get a match we will use the root - return 0 + return self.recursionFolder(client, 0, folder) def download(self, data = None, media = None, filedata = None): if not media: media = {} From 2277322e572b95aa61557e3b10e99de97a993e4f Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 10 Feb 2015 13:47:22 +0100 Subject: [PATCH 37/60] Traceback import missing --- couchpotato/core/notifications/webhook.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/couchpotato/core/notifications/webhook.py b/couchpotato/core/notifications/webhook.py index d970bb53..8dc63291 100644 --- a/couchpotato/core/notifications/webhook.py +++ b/couchpotato/core/notifications/webhook.py @@ -1,15 +1,17 @@ +import traceback + from couchpotato.core.helpers.encoding import toUnicode from couchpotato.core.helpers.variable import getIdentifier from couchpotato.core.logger import CPLog from couchpotato.core.notifications.base import Notification + log = CPLog(__name__) autoload = 'Webhook' class Webhook(Notification): - def notify(self, message = '', data = None, listener = None): if not data: data = {} From 0fd01aa6974f241aab32b2b7d2bafa674a6f7640 Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 10 Feb 2015 14:01:51 +0100 Subject: [PATCH 38/60] Cleanup --- .../media/movie/providers/automation/crowdai.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/couchpotato/core/media/movie/providers/automation/crowdai.py b/couchpotato/core/media/movie/providers/automation/crowdai.py index 0d33f11b..56f1fb27 100644 --- a/couchpotato/core/media/movie/providers/automation/crowdai.py +++ b/couchpotato/core/media/movie/providers/automation/crowdai.py @@ -1,5 +1,3 @@ -from xml.etree.ElementTree import QName -import datetime import re from couchpotato.core.helpers.rss import RSS @@ -21,7 +19,6 @@ class CrowdAI(Automation, RSS): movies = [] - newznab_namespace = 'http://www.newznab.com/DTD/2010/feeds/attributes/' urls = dict(zip(splitString(self.conf('automation_urls')), [tryInt(x) for x in splitString(self.conf('automation_urls_use'))])) for url in urls: @@ -33,20 +30,18 @@ class CrowdAI(Automation, RSS): for movie in rss_movies: - title = "" - description = self.getTextElement(movie, "description") + description = self.getTextElement(movie, 'description') grabs = 0 - + for item in movie: if item.attrib.get('name') == 'grabs': grabs = item.attrib.get('value') break - - + if int(grabs) > tryInt(self.conf('number_grabs')): - title = re.match( r'.*Title: .a href.*/">(.*) \(\d{4}\).*', description).group(1) + title = re.match(r'.*Title: .a href.*/">(.*) \(\d{4}\).*', description).group(1) log.info2('%s grabs for movie: %s, enqueue...', (grabs, title)) - year = re.match( r'.*Year: (\d{4}).*', description).group(1) + year = re.match(r'.*Year: (\d{4}).*', description).group(1) imdb = self.search(title, year) if imdb and self.isMinimalMovie(imdb): @@ -80,7 +75,7 @@ config = [{ 'label': 'url', 'type': 'combined', 'combine': ['automation_urls_use', 'automation_urls'], - 'default': 'http://YOUR_PROVIDER/rss?t=THE_MOVIE_CATEGORY&i=YOUR_USER_ID&r=YOUR_API_KEY&res=2&rls=2&num=100', + 'default': 'http://YOUR_PROVIDER/rss?t=THE_MOVIE_CATEGORY&i=YOUR_USER_ID&r=YOUR_API_KEY&res=2&rls=2&num=100', }, { 'name': 'number_grabs', From 9b91d1d6c0c0274fe3d94c25a3fb908ce65814ab Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 10 Feb 2015 14:10:55 +0100 Subject: [PATCH 39/60] Remove favor, link to api key page --- .../media/_base/providers/torrent/hdaccess.py | 22 +------------------ 1 file changed, 1 insertion(+), 21 deletions(-) diff --git a/couchpotato/core/media/_base/providers/torrent/hdaccess.py b/couchpotato/core/media/_base/providers/torrent/hdaccess.py index 811c599c..1a6b0edd 100644 --- a/couchpotato/core/media/_base/providers/torrent/hdaccess.py +++ b/couchpotato/core/media/_base/providers/torrent/hdaccess.py @@ -1,5 +1,4 @@ import re -import json import traceback from couchpotato.core.helpers.variable import tryInt, getIdentifier @@ -42,17 +41,6 @@ class Base(TorrentProvider): torrent_desc += '/ Internal' torrentscore += 200 - if resolution == '1080p': - torrentscore += 100 - if encoding == 'x264' and self.conf('favor') in ['encode', 'all']: - torrentscore += 100 - elif encoding == 'Encode' and self.conf('favor') in ['encode', 'all']: - torrentscore += 100 - elif encoding == 'Remux' and self.conf('favor') in ['remux', 'all']: - torrentscore += 200 - elif encoding == 'Bluray' and self.conf('favor') in ['bluray', 'all']: - torrentscore += 300 - if seeders == 0: torrentscore = 0 @@ -97,7 +85,7 @@ config = [{ 'name': 'apikey', 'default': '', 'label': 'API Key', - 'description': 'Enter your site api key. This can be find in \'Edit My Profile\'->Security', + 'description': 'Enter your site api key. This can be find on Profile Security', }, { 'name': 'seed_ratio', @@ -120,14 +108,6 @@ config = [{ 'default': 1, 'description': 'Favors internal releases over non-internal releases.', }, - { - 'name': 'favor', - 'advanced': True, - 'default': 'all', - 'type': 'dropdown', - 'values': [('Blurays & Encodes & Remuxes', 'all'), ('Blurays', 'bluray'), ('Encodes', 'encode'), ('Remuxes', 'remux'), ('None', 'none')], - 'description': 'Give extra scoring to blurays(+300), remuxes(+200) or encodes(+100).', - }, { 'name': 'internal_only', 'advanced': True, From ce768f45c51267f48ba40202ec60cf36db4195cc Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 10 Feb 2015 14:36:54 +0100 Subject: [PATCH 40/60] Make RottenTomato logging more clear close #4618 --- .../core/media/movie/providers/automation/rottentomatoes.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/couchpotato/core/media/movie/providers/automation/rottentomatoes.py b/couchpotato/core/media/movie/providers/automation/rottentomatoes.py index a01f76d2..65d54f13 100644 --- a/couchpotato/core/media/movie/providers/automation/rottentomatoes.py +++ b/couchpotato/core/media/movie/providers/automation/rottentomatoes.py @@ -39,15 +39,14 @@ class Rottentomatoes(Automation, RSS): if result: - log.info2('Something smells...') rating = tryInt(self.getTextElement(movie, rating_tag)) name = result.group(0) + print rating, tryInt(self.conf('tomatometer_percent')) if rating < tryInt(self.conf('tomatometer_percent')): log.info2('%s seems to be rotten...', name) else: - - log.info2('Found %s fresh enough movies, enqueuing: %s', (rating, name)) + log.info2('Found %s with fresh rating %s', (name, rating)) year = datetime.datetime.now().strftime("%Y") imdb = self.search(name, year) From 6dcb3f3bf20fa05cfdeec53b7c44913423a90ce5 Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 10 Feb 2015 14:55:22 +0100 Subject: [PATCH 41/60] Change bitsoup category id fixes #4629 --- couchpotato/core/media/movie/providers/torrent/bitsoup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/media/movie/providers/torrent/bitsoup.py b/couchpotato/core/media/movie/providers/torrent/bitsoup.py index e9d69fe5..b0c8eded 100644 --- a/couchpotato/core/media/movie/providers/torrent/bitsoup.py +++ b/couchpotato/core/media/movie/providers/torrent/bitsoup.py @@ -11,7 +11,7 @@ autoload = 'Bitsoup' class Bitsoup(MovieProvider, Base): cat_ids = [ ([17], ['3d']), - ([41], ['720p', '1080p']), + ([80], ['720p', '1080p']), ([20], ['dvdr']), ([19], ['brrip', 'dvdrip']), ] From 11b9bc39abac6e47f6565814954d060e0291bf2b Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 10 Feb 2015 15:40:55 +0100 Subject: [PATCH 42/60] Show tried to often error for TD --- couchpotato/core/media/_base/providers/torrent/torrentday.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/couchpotato/core/media/_base/providers/torrent/torrentday.py b/couchpotato/core/media/_base/providers/torrent/torrentday.py index 57224ffe..73014339 100644 --- a/couchpotato/core/media/_base/providers/torrent/torrentday.py +++ b/couchpotato/core/media/_base/providers/torrent/torrentday.py @@ -1,3 +1,4 @@ +import re from couchpotato.core.helpers.variable import tryInt from couchpotato.core.logger import CPLog from couchpotato.core.media._base.providers.torrent.base import TorrentProvider @@ -55,6 +56,10 @@ class Base(TorrentProvider): } def loginSuccess(self, output): + often = re.search('You tried too often, please wait .*', output) + if often: + raise Exception(often.group(0)[:-6].strip()) + return 'Password not correct' not in output def loginCheckSuccess(self, output): From b1fc8ad86252ec525ac85df53c3b2f73358d5ece Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 10 Feb 2015 16:21:32 +0100 Subject: [PATCH 43/60] Letterboxed new html markup fix #4640 --- .../core/media/movie/providers/automation/letterboxd.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/couchpotato/core/media/movie/providers/automation/letterboxd.py b/couchpotato/core/media/movie/providers/automation/letterboxd.py index e9fc8741..d43821c0 100644 --- a/couchpotato/core/media/movie/providers/automation/letterboxd.py +++ b/couchpotato/core/media/movie/providers/automation/letterboxd.py @@ -48,11 +48,12 @@ class Letterboxd(Automation): soup = BeautifulSoup(self.getHTMLData(self.url % username)) - for movie in soup.find_all('a', attrs = {'class': 'frame'}): - match = removeEmpty(self.pattern.split(movie['title'])) + for movie in soup.find_all('li', attrs = {'class': 'poster-container'}): + img = movie.find('img', movie) + title = img.get('alt') + movies.append({ - 'title': match[0], - 'year': match[1] + 'title': title }) return movies From 920d3cb44e528e8c8463531897790ae5bf51cd92 Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 10 Feb 2015 16:27:13 +0100 Subject: [PATCH 44/60] Don't verify SYNO downloader thingymajig fix #4641 --- couchpotato/core/downloaders/synology.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/downloaders/synology.py b/couchpotato/core/downloaders/synology.py index b5327ccb..8368eb1f 100644 --- a/couchpotato/core/downloaders/synology.py +++ b/couchpotato/core/downloaders/synology.py @@ -137,7 +137,7 @@ class SynologyRPC(object): def _req(self, url, args, files = None): response = {'success': False} try: - req = requests.post(url, data = args, files = files) + req = requests.post(url, data = args, files = files, verify = False) req.raise_for_status() response = json.loads(req.text) if response['success']: From afc9039625c3f85c0a12699e45acd19dd03fc67e Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 10 Feb 2015 16:50:53 +0100 Subject: [PATCH 45/60] Also search lower qualities on OMGWTF fix #4527 --- couchpotato/core/media/_base/providers/nzb/omgwtfnzbs.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/couchpotato/core/media/_base/providers/nzb/omgwtfnzbs.py b/couchpotato/core/media/_base/providers/nzb/omgwtfnzbs.py index ea5f90f7..3d002fb8 100644 --- a/couchpotato/core/media/_base/providers/nzb/omgwtfnzbs.py +++ b/couchpotato/core/media/_base/providers/nzb/omgwtfnzbs.py @@ -18,20 +18,13 @@ class Base(NZBProvider, RSS): http_time_between_calls = 1 # Seconds cat_ids = [ - ([15], ['dvdrip']), + ([15], ['dvdrip', 'scr', 'r5', 'tc', 'ts', 'cam']), ([15, 16], ['brrip']), ([16], ['720p', '1080p', 'bd50']), ([17], ['dvdr']), ] cat_backup_id = 'movie' - def search(self, movie, quality): - - if quality['identifier'] in fireEvent('quality.pre_releases', single = True): - return [] - - return super(Base, self).search(movie, quality) - def _searchOnTitle(self, title, movie, quality, results): q = '%s %s' % (title, movie['info']['year']) From 9f77597c11ba5da93a35a49386c11e1deb3710a8 Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 10 Feb 2015 17:15:53 +0100 Subject: [PATCH 46/60] Torrentz search on title fix #4510 --- couchpotato/core/media/_base/providers/torrent/torrentz.py | 4 ++-- couchpotato/core/media/movie/providers/torrent/torrentz.py | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/couchpotato/core/media/_base/providers/torrent/torrentz.py b/couchpotato/core/media/_base/providers/torrent/torrentz.py index 8a5455c9..a5c1ed05 100644 --- a/couchpotato/core/media/_base/providers/torrent/torrentz.py +++ b/couchpotato/core/media/_base/providers/torrent/torrentz.py @@ -22,12 +22,12 @@ class Base(TorrentMagnetProvider, RSS): http_time_between_calls = 0 - def _search(self, media, quality, results): + def _searchOnTitle(self, title, media, quality, results): search_url = self.urls['verified_search'] if self.conf('verified_only') else self.urls['search'] # Create search parameters - search_params = self.buildUrl(media) + search_params = self.buildUrl(title, media, quality) smin = quality.get('size_min') smax = quality.get('size_max') diff --git a/couchpotato/core/media/movie/providers/torrent/torrentz.py b/couchpotato/core/media/movie/providers/torrent/torrentz.py index 742554c4..011ec430 100644 --- a/couchpotato/core/media/movie/providers/torrent/torrentz.py +++ b/couchpotato/core/media/movie/providers/torrent/torrentz.py @@ -1,6 +1,5 @@ from couchpotato.core.helpers.encoding import tryUrlencode from couchpotato.core.logger import CPLog -from couchpotato.core.event import fireEvent from couchpotato.core.media._base.providers.torrent.torrentz import Base from couchpotato.core.media.movie.providers.base import MovieProvider @@ -11,5 +10,5 @@ autoload = 'Torrentz' class Torrentz(MovieProvider, Base): - def buildUrl(self, media): - return tryUrlencode('"%s"' % fireEvent('library.query', media, single = True)) + def buildUrl(self, title, media, quality): + return tryUrlencode('"%s %s"' % (title, media['info']['year'])) \ No newline at end of file From debd1855ddeabe6dd75997b0cf9f345d7ad11735 Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 10 Feb 2015 20:47:19 +0100 Subject: [PATCH 47/60] Move Yify to v2 --- .../media/_base/providers/torrent/yify.py | 45 ++++++++++--------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/couchpotato/core/media/_base/providers/torrent/yify.py b/couchpotato/core/media/_base/providers/torrent/yify.py index 725aabbc..b6b91e29 100644 --- a/couchpotato/core/media/_base/providers/torrent/yify.py +++ b/couchpotato/core/media/_base/providers/torrent/yify.py @@ -2,27 +2,25 @@ import traceback from couchpotato.core.helpers.variable import tryInt, getIdentifier from couchpotato.core.logger import CPLog -from couchpotato.core.media._base.providers.torrent.base import TorrentMagnetProvider +from couchpotato.core.media._base.providers.torrent.base import TorrentProvider log = CPLog(__name__) -class Base(TorrentMagnetProvider): +class Base(TorrentProvider): urls = { - 'test': '%s/api', - 'search': '%s/api/list.json?keywords=%s', - 'detail': '%s/api/movie.json?id=%s' + 'test': '%s/api/v2', + 'search': '%s/api/v2/list_movies.json?limit=50&query_term=%s' } http_time_between_calls = 1 # seconds proxy_list = [ 'https://yts.re', - 'http://ytsproxy.come.in', + 'https://yts.wf', 'http://yts.im', - 'http://yify-torrents.im', ] def search(self, movie, quality): @@ -41,25 +39,28 @@ class Base(TorrentMagnetProvider): search_url = self.urls['search'] % (domain, getIdentifier(movie)) data = self.getJsonData(search_url) + data = data.get('data') - if data and data.get('MovieList'): + if data and data.get('movies'): try: - for result in data.get('MovieList'): + for result in data.get('movies'): - if result['Quality'] and result['Quality'] not in result['MovieTitle']: - title = result['MovieTitle'] + ' BrRip ' + result['Quality'] - else: - title = result['MovieTitle'] + ' BrRip' + for release in result.get('torrents', []): - results.append({ - 'id': result['MovieID'], - 'name': title, - 'url': result['TorrentMagnetUrl'], - 'detail_url': self.urls['detail'] % (domain, result['MovieID']), - 'size': self.parseSize(result['Size']), - 'seeders': tryInt(result['TorrentSeeds']), - 'leechers': tryInt(result['TorrentPeers']), - }) + if release['quality'] and release['quality'] not in result['title_long']: + title = result['title_long'] + ' BRRip ' + release['quality'] + else: + title = result['title_long'] + ' BRRip' + + results.append({ + 'id': release['hash'], + 'name': title, + 'url': release['url'], + 'detail_url': result['url'], + 'size': self.parseSize(release['size']), + 'seeders': tryInt(release['seeds']), + 'leechers': tryInt(release['peers']), + }) except: log.error('Failed getting results from %s: %s', (self.getName(), traceback.format_exc())) From 94c3969f100c7470af467d3176b630af186345fb Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 10 Feb 2015 20:52:15 +0100 Subject: [PATCH 48/60] Use https for yify proxy --- couchpotato/core/media/_base/providers/torrent/yify.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/media/_base/providers/torrent/yify.py b/couchpotato/core/media/_base/providers/torrent/yify.py index b6b91e29..492eeb65 100644 --- a/couchpotato/core/media/_base/providers/torrent/yify.py +++ b/couchpotato/core/media/_base/providers/torrent/yify.py @@ -20,7 +20,7 @@ class Base(TorrentProvider): proxy_list = [ 'https://yts.re', 'https://yts.wf', - 'http://yts.im', + 'https://yts.im', ] def search(self, movie, quality): From 427c77a9efba5838679cdc0c87cc684d4260b4b1 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 15 Feb 2015 19:23:45 +0100 Subject: [PATCH 49/60] Remove podnapisi --- libs/subliminal/services/podnapisi.py | 110 -------------------------- 1 file changed, 110 deletions(-) delete mode 100755 libs/subliminal/services/podnapisi.py diff --git a/libs/subliminal/services/podnapisi.py b/libs/subliminal/services/podnapisi.py deleted file mode 100755 index 618c0e77..00000000 --- a/libs/subliminal/services/podnapisi.py +++ /dev/null @@ -1,110 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2011-2012 Antoine Bertin -# -# This file is part of subliminal. -# -# subliminal is free software; you can redistribute it and/or modify it under -# the terms of the GNU Lesser General Public License as published by -# the Free Software Foundation; either version 3 of the License, or -# (at your option) any later version. -# -# subliminal is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with subliminal. If not, see . -from . import ServiceBase -from ..exceptions import ServiceError, DownloadFailedError -from ..language import language_set, Language -from ..subtitles import get_subtitle_path, ResultSubtitle -from ..utils import to_unicode -from ..videos import Episode, Movie -from hashlib import md5, sha256 -import logging -import xmlrpclib - - -logger = logging.getLogger(__name__) - - -class Podnapisi(ServiceBase): - server_url = 'http://ssp.podnapisi.net:8000' - api_based = True - languages = language_set(['ar', 'be', 'bg', 'bs', 'ca', 'ca', 'cs', 'da', 'de', 'el', 'en', - 'es', 'et', 'fa', 'fi', 'fr', 'ga', 'he', 'hi', 'hr', 'hu', 'id', - 'is', 'it', 'ja', 'ko', 'lt', 'lv', 'mk', 'ms', 'nl', 'nn', 'pl', - 'pt', 'ro', 'ru', 'sk', 'sl', 'sq', 'sr', 'sv', 'th', 'tr', 'uk', - 'vi', 'zh', 'es-ar', 'pt-br']) - language_map = {'jp': Language('jpn'), Language('jpn'): 'jp', - 'gr': Language('gre'), Language('gre'): 'gr', - 'pb': Language('por-BR'), Language('por-BR'): 'pb', - 'ag': Language('spa-AR'), Language('spa-AR'): 'ag', - 'cyr': Language('srp')} - videos = [Episode, Movie] - require_video = True - - def __init__(self, config=None): - super(Podnapisi, self).__init__(config) - self.server = xmlrpclib.ServerProxy(self.server_url) - self.token = None - - def init(self): - super(Podnapisi, self).init() - result = self.server.initiate(self.user_agent) - if result['status'] != 200: - raise ServiceError('Initiate failed') - username = 'python_subliminal' - password = sha256(md5('XWFXQ6gE5Oe12rv4qxXX').hexdigest() + result['nonce']).hexdigest() - self.token = result['session'] - result = self.server.authenticate(self.token, username, password) - if result['status'] != 200: - raise ServiceError('Authenticate failed') - - def terminate(self): - super(Podnapisi, self).terminate() - - def query(self, filepath, languages, moviehash): - results = self.server.search(self.token, [moviehash]) - if results['status'] != 200: - logger.error('Search failed with error code %d' % results['status']) - return [] - if not results['results'] or not results['results'][moviehash]['subtitles']: - logger.debug(u'Could not find subtitles for %r with token %s' % (moviehash, self.token)) - return [] - subtitles = [] - for result in results['results'][moviehash]['subtitles']: - language = self.get_language(result['lang']) - if language not in languages: - continue - path = get_subtitle_path(filepath, language, self.config.multi) - subtitle = ResultSubtitle(path, language, self.__class__.__name__.lower(), result['id'], - release=to_unicode(result['release']), confidence=result['weight']) - subtitles.append(subtitle) - if not subtitles: - return [] - # Convert weight to confidence - max_weight = float(max([s.confidence for s in subtitles])) - min_weight = float(min([s.confidence for s in subtitles])) - for subtitle in subtitles: - if max_weight == 0 and min_weight == 0: - subtitle.confidence = 1.0 - else: - subtitle.confidence = (subtitle.confidence - min_weight) / (max_weight - min_weight) - return subtitles - - def list_checked(self, video, languages): - results = self.query(video.path, languages, video.hashes['OpenSubtitles']) - return results - - def download(self, subtitle): - results = self.server.download(self.token, [subtitle.link]) - if results['status'] != 200: - raise DownloadFailedError() - subtitle.link = 'http://www.podnapisi.net/static/podnapisi/' + results['names'][0]['filename'] - self.download_file(subtitle.link, subtitle.path) - return subtitle - - -Service = Podnapisi From 7db8b233c82859092482b9db0be81d42a12639ae Mon Sep 17 00:00:00 2001 From: Ruud Date: Wed, 18 Feb 2015 17:21:24 +0100 Subject: [PATCH 50/60] Don't decode string if confidence isn't high enough --- couchpotato/core/helpers/encoding.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/couchpotato/core/helpers/encoding.py b/couchpotato/core/helpers/encoding.py index c28b6974..c2790060 100644 --- a/couchpotato/core/helpers/encoding.py +++ b/couchpotato/core/helpers/encoding.py @@ -38,9 +38,12 @@ def toUnicode(original, *args): try: detected = detect(original) try: - return original.decode(detected.get('encoding')) + if detected.get('confidence') > 0.8: + return original.decode(detected.get('encoding')) except: - return ek(original, *args) + pass + + return ek(original, *args) except: raise except: From 2dc1c1dd388cb18716fdb02b6c4c6bbc44bcd107 Mon Sep 17 00:00:00 2001 From: peerster Date: Thu, 19 Feb 2015 20:07:22 +0100 Subject: [PATCH 51/60] Update torrentshack with new URL --- .../media/_base/providers/torrent/torrentshack.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/couchpotato/core/media/_base/providers/torrent/torrentshack.py b/couchpotato/core/media/_base/providers/torrent/torrentshack.py index b65222b3..71f4e625 100644 --- a/couchpotato/core/media/_base/providers/torrent/torrentshack.py +++ b/couchpotato/core/media/_base/providers/torrent/torrentshack.py @@ -13,12 +13,12 @@ log = CPLog(__name__) class Base(TorrentProvider): urls = { - 'test': 'http://torrentshack.eu/', - 'login': 'http://torrentshack.eu/login.php', - 'login_check': 'http://torrentshack.eu/inbox.php', - 'detail': 'http://torrentshack.eu/torrent/%s', - 'search': 'http://torrentshack.eu/torrents.php?action=advanced&searchstr=%s&scene=%s&filter_cat[%d]=1', - 'download': 'http://torrentshack.eu/%s', + 'test': 'https://theshack.us.to/', + 'login': 'https://theshack.us.to/login.php', + 'login_check': 'https://theshack.us.to/inbox.php', + 'detail': 'https://theshack.us.to/torrent/%s', + 'search': 'https://theshack.us.to/torrents.php?action=advanced&searchstr=%s&scene=%s&filter_cat[%d]=1', + 'download': 'https://theshack.us.to/%s', } http_time_between_calls = 1 # Seconds From b19b0775c7ca540c42a2fe9ed7aa36a43767689d Mon Sep 17 00:00:00 2001 From: Ruud Date: Fri, 20 Feb 2015 22:16:12 +0100 Subject: [PATCH 52/60] Force update to new poster on refresh fix #4671 --- couchpotato/core/media/__init__.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/couchpotato/core/media/__init__.py b/couchpotato/core/media/__init__.py index 7a178b85..0d98600d 100755 --- a/couchpotato/core/media/__init__.py +++ b/couchpotato/core/media/__init__.py @@ -1,9 +1,10 @@ import os import traceback -from couchpotato import CPLog +from couchpotato import CPLog, md5 from couchpotato.core.event import addEvent, fireEvent, fireEventAsync from couchpotato.core.helpers.encoding import toUnicode +from couchpotato.core.helpers.variable import getExt from couchpotato.core.plugins.base import Plugin import six @@ -92,7 +93,15 @@ class MediaBase(Plugin): if not isinstance(image, (str, unicode)): continue - if file_type not in existing_files or len(existing_files.get(file_type, [])) == 0: + # Check if it has top image + filename = '%s.%s' % (md5(image), getExt(image)) + existing = existing_files.get(file_type, []) + has_latest = False + for x in existing: + if filename in x: + has_latest = True + + if not has_latest or file_type not in existing_files or len(existing_files.get(file_type, [])) == 0: file_path = fireEvent('file.download', url = image, single = True) if file_path: existing_files[file_type] = [toUnicode(file_path)] From f8631c6d536ec19ac5e6ecadee42d00006c0d8b4 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 21 Feb 2015 21:29:37 +0100 Subject: [PATCH 53/60] Add extra category for TorrentLeech fix #4683 --- .../core/media/_base/providers/torrent/torrentleech.py | 2 +- .../core/media/movie/providers/torrent/torrentleech.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/couchpotato/core/media/_base/providers/torrent/torrentleech.py b/couchpotato/core/media/_base/providers/torrent/torrentleech.py index 83eb5f1f..3daa10b2 100644 --- a/couchpotato/core/media/_base/providers/torrent/torrentleech.py +++ b/couchpotato/core/media/_base/providers/torrent/torrentleech.py @@ -17,7 +17,7 @@ class Base(TorrentProvider): 'login': 'https://www.torrentleech.org/user/account/login/', 'login_check': 'https://torrentleech.org/user/messages', 'detail': 'https://www.torrentleech.org/torrent/%s', - 'search': 'https://www.torrentleech.org/torrents/browse/index/query/%s/categories/%d', + 'search': 'https://www.torrentleech.org/torrents/browse/index/query/%s/categories/%s', 'download': 'https://www.torrentleech.org%s', } diff --git a/couchpotato/core/media/movie/providers/torrent/torrentleech.py b/couchpotato/core/media/movie/providers/torrent/torrentleech.py index d72f4257..eea74f80 100644 --- a/couchpotato/core/media/movie/providers/torrent/torrentleech.py +++ b/couchpotato/core/media/movie/providers/torrent/torrentleech.py @@ -16,12 +16,12 @@ class TorrentLeech(MovieProvider, Base): ([9], ['ts', 'tc']), ([10], ['r5', 'scr']), ([11], ['dvdrip']), - ([14], ['brrip']), + ([13, 14], ['brrip']), ([12], ['dvdr']), ] def buildUrl(self, title, media, quality): return ( tryUrlencode(title.replace(':', '')), - self.getCatId(quality)[0] + ','.join([str(x) for x in self.getCatId(quality)]) ) From 84a458d40b34de64a2d538a037b0d54e6d765d96 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 22 Feb 2015 13:06:29 +0100 Subject: [PATCH 54/60] Add user-agent and type to omdbapi --- couchpotato/core/media/movie/providers/info/omdbapi.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/couchpotato/core/media/movie/providers/info/omdbapi.py b/couchpotato/core/media/movie/providers/info/omdbapi.py index d3a83b63..16f30adc 100644 --- a/couchpotato/core/media/movie/providers/info/omdbapi.py +++ b/couchpotato/core/media/movie/providers/info/omdbapi.py @@ -2,6 +2,7 @@ import json import re import traceback +from couchpotato import Env from couchpotato.core.event import addEvent, fireEvent from couchpotato.core.helpers.encoding import tryUrlencode from couchpotato.core.helpers.variable import tryInt, tryFloat, splitString @@ -17,8 +18,8 @@ autoload = 'OMDBAPI' class OMDBAPI(MovieProvider): urls = { - 'search': 'http://www.omdbapi.com/?%s', - 'info': 'http://www.omdbapi.com/?i=%s', + 'search': 'http://www.omdbapi.com/?type=movie&%s', + 'info': 'http://www.omdbapi.com/?type=movie&i=%s', } http_time_between_calls = 0 @@ -38,7 +39,8 @@ class OMDBAPI(MovieProvider): } cache_key = 'omdbapi.cache.%s' % q - cached = self.getCache(cache_key, self.urls['search'] % tryUrlencode({'t': name_year.get('name'), 'y': name_year.get('year', '')}), timeout = 3) + url = self.urls['search'] % tryUrlencode({'t': name_year.get('name'), 'y': name_year.get('year', '')}) + cached = self.getCache(cache_key, url, timeout = 3, headers = {'User-Agent': Env.getIdentifier()}) if cached: result = self.parseMovie(cached) @@ -56,7 +58,7 @@ class OMDBAPI(MovieProvider): return {} cache_key = 'omdbapi.cache.%s' % identifier - cached = self.getCache(cache_key, self.urls['info'] % identifier, timeout = 3) + cached = self.getCache(cache_key, self.urls['info'] % identifier, timeout = 3, headers = {'User-Agent': Env.getIdentifier()}) if cached: result = self.parseMovie(cached) From 99a06212387814f42922c6e8957bbe60e3a505a0 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 22 Feb 2015 14:30:50 +0100 Subject: [PATCH 55/60] Use keep-alive connection --- couchpotato/core/plugins/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/plugins/base.py b/couchpotato/core/plugins/base.py index e4b27c9b..6d63b83c 100644 --- a/couchpotato/core/plugins/base.py +++ b/couchpotato/core/plugins/base.py @@ -196,7 +196,7 @@ class Plugin(object): headers['Host'] = headers.get('Host', None) headers['User-Agent'] = headers.get('User-Agent', self.user_agent) headers['Accept-encoding'] = headers.get('Accept-encoding', 'gzip') - headers['Connection'] = headers.get('Connection', 'close') + headers['Connection'] = headers.get('Connection', 'keep-alive') headers['Cache-Control'] = headers.get('Cache-Control', 'max-age=0') r = Env.get('http_opener') From 6b8458d87f8f1b3b527300b3c10c6ff174a02015 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 22 Feb 2015 14:49:37 +0100 Subject: [PATCH 56/60] Hadouken apikey check not using correct settingskey fix #4674 --- couchpotato/core/downloaders/hadouken.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/downloaders/hadouken.py b/couchpotato/core/downloaders/hadouken.py index c7dddbe7..c89ed0e7 100644 --- a/couchpotato/core/downloaders/hadouken.py +++ b/couchpotato/core/downloaders/hadouken.py @@ -31,7 +31,7 @@ class Hadouken(DownloaderBase): log.error('Config properties are not filled in correctly, port is missing.') return False - if not self.conf('apikey'): + if not self.conf('api_key'): log.error('Config properties are not filled in correctly, API key is missing.') return False From 6598f53fd48df344fac30a71c7a16aaab45acfd8 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 22 Feb 2015 15:55:54 +0100 Subject: [PATCH 57/60] Quality check improve --- couchpotato/core/plugins/quality/main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/couchpotato/core/plugins/quality/main.py b/couchpotato/core/plugins/quality/main.py index 96fb1a3a..7d80f728 100644 --- a/couchpotato/core/plugins/quality/main.py +++ b/couchpotato/core/plugins/quality/main.py @@ -240,7 +240,7 @@ class QualityPlugin(Plugin): # Add additional size score if only 1 size validated if len(size_scores) == 1: - self.calcScore(score, size_scores[0], 8) + self.calcScore(score, size_scores[0], 7) del size_scores # Return nothing if all scores are <= 0 @@ -491,6 +491,7 @@ class QualityPlugin(Plugin): 'Movie Name.2014.720p Web-Dl Aac2.0 h264-ReleaseGroup': {'size': 3800, 'quality': 'brrip'}, 'Movie Name.2014.720p.WEBRip.x264.AC3-ReleaseGroup': {'size': 3000, 'quality': 'scr'}, 'Movie.Name.2014.1080p.HDCAM.-.ReleaseGroup': {'size': 5300, 'quality': 'cam'}, + 'Movie.Name.2014.720p.HDSCR.4PARTS.MP4.AAC.ReleaseGroup': {'size': 2401, 'quality': 'scr'}, } correct = 0 From 0d6c3c8ecb270f04cf336662f8f61666dcdf4b76 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 22 Feb 2015 16:06:07 +0100 Subject: [PATCH 58/60] Yify, only use data when available --- couchpotato/core/media/_base/providers/torrent/yify.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/media/_base/providers/torrent/yify.py b/couchpotato/core/media/_base/providers/torrent/yify.py index 492eeb65..4071a156 100644 --- a/couchpotato/core/media/_base/providers/torrent/yify.py +++ b/couchpotato/core/media/_base/providers/torrent/yify.py @@ -41,7 +41,7 @@ class Base(TorrentProvider): data = self.getJsonData(search_url) data = data.get('data') - if data and data.get('movies'): + if isinstance(data, dict) and data.get('movies'): try: for result in data.get('movies'): From 0f82cda811c635d71d0d539b73f56fabd257431f Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 22 Feb 2015 16:09:22 +0100 Subject: [PATCH 59/60] Remove podnapisi from subtile list --- couchpotato/core/plugins/subtitle.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/plugins/subtitle.py b/couchpotato/core/plugins/subtitle.py index 1766088e..e8baef0d 100644 --- a/couchpotato/core/plugins/subtitle.py +++ b/couchpotato/core/plugins/subtitle.py @@ -16,7 +16,7 @@ autoload = 'Subtitle' class Subtitle(Plugin): - services = ['opensubtitles', 'thesubdb', 'subswiki', 'podnapisi', 'subscenter'] + services = ['opensubtitles', 'thesubdb', 'subswiki', 'subscenter'] def __init__(self): addEvent('renamer.before', self.searchSingle) From adb744a526b9e6a7ac9fed905ffade4a42803e12 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 22 Feb 2015 17:42:29 +0100 Subject: [PATCH 60/60] Don't show double updater type --- couchpotato/static/scripts/page/about.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/static/scripts/page/about.js b/couchpotato/static/scripts/page/about.js index a2482f8b..f36f7e48 100644 --- a/couchpotato/static/scripts/page/about.js +++ b/couchpotato/static/scripts/page/about.js @@ -117,7 +117,7 @@ var AboutSettingTab = new Class({ var self = this; var date = new Date(json.version.date * 1000); self.version_text.set('text', json.version.hash + (json.version.date ? ' ('+date.toLocaleString()+')' : '')); - self.updater_type.set('text', json.version.type + ', ' + json.branch); + self.updater_type.set('text', (json.version.type != json.branch) ? (json.version.type + ', ' + json.branch) : json.branch); } });