Merge branch 'develop' into redesign

couchpotato/core/downloaders/hadouken.py

@@ -31,7 +31,7 @@ class Hadouken(DownloaderBase):
             log.error('Config properties are not filled in correctly, port is missing.')
             return False
 
-        if not self.conf('apikey'):
+        if not self.conf('api_key'):
             log.error('Config properties are not filled in correctly, API key is missing.')
             return False
 

couchpotato/core/downloaders/putio/__init__.py

@@ -28,6 +28,11 @@ config = [{
                     'description': 'This is the OAUTH_TOKEN from your putio API',
                     'advanced': True,
                 },
+                {
+                    'name': 'folder',
+                    'description': ('The folder on putio where you want the upload to go','Will find the first first folder that matches this name'),
+                    'default': 0,
+                },
                 {
                     'name': 'callback_host',
                     'description': 'External reachable url to CP so put.io can do it\'s thing',

couchpotato/core/downloaders/putio/main.py

@@ -28,20 +28,41 @@ class PutIO(DownloaderBase):
 
         return super(PutIO, self).__init__()
 
+    # This is a recusive function to check for the folders
+    def recursionFolder(self, client, folder = 0, tfolder = ''):
+        files = client.File.list(folder)
+        for f in files:
+            if f.content_type == 'application/x-directory':
+                if f.name == tfolder:
+                   return f.id
+                else:
+                    result = self.recursionFolder(client, f.id, tfolder)
+                    if result != 0:
+                       return result
+        return 0
+
+    # This will check the root for the folder, and kick of recusively checking sub folder
+    def convertFolder(self, client, folder):
+        if folder == 0:
+            return 0
+        else:
+            return self.recursionFolder(client, 0, folder)
+
     def download(self, data = None, media = None, filedata = None):
         if not media: media = {}
         if not data: data = {}
 
         log.info('Sending "%s" to put.io', data.get('name'))
         url = data.get('url')
-
         client = pio.Client(self.conf('oauth_token'))
+        putioFolder = self.convertFolder(client, self.conf('folder'))
+        log.debug('putioFolder ID is %s', putioFolder)
         # It might be possible to call getFromPutio from the renamer if we can then we don't need to do this.
         # Note callback_host is NOT our address, it's the internet host that putio can call too
         callbackurl = None
         if self.conf('download'):
-            callbackurl = 'http://' + self.conf('callback_host') + '/' + '%sdownloader.putio.getfrom/' %Env.get('api_base'.strip('/'))
-        resp = client.Transfer.add_url(url, callback_url = callbackurl)
+            callbackurl = 'http://' + self.conf('callback_host') + '%sdownloader.putio.getfrom/' %Env.get('api_base'.strip('/'))
+        resp = client.Transfer.add_url(url, callback_url = callbackurl, parent_id = putioFolder)
         log.debug('resp is %s', resp.id);
         return self.downloadReturnId(resp.id)
 
@@ -124,7 +145,9 @@ class PutIO(DownloaderBase):
         client = pio.Client(self.conf('oauth_token'))
 
         log.debug('About to get file List')
-        files = client.File.list()
+        putioFolder = self.convertFolder(client, self.conf('folder'))
+        log.debug('PutioFolderID is %s', putioFolder)
+        files = client.File.list(parent_id=putioFolder)
         downloaddir = self.conf('download_dir')
 
         for f in files:

couchpotato/core/downloaders/synology.py

@@ -137,7 +137,7 @@ class SynologyRPC(object):
     def _req(self, url, args, files = None):
         response = {'success': False}
         try:
-            req = requests.post(url, data = args, files = files)
+            req = requests.post(url, data = args, files = files, verify = False)
             req.raise_for_status()
             response = json.loads(req.text)
             if response['success']:

couchpotato/core/helpers/encoding.py

@@ -38,9 +38,12 @@ def toUnicode(original, *args):
                 try:
                     detected = detect(original)
                     try:
-                        return original.decode(detected.get('encoding'))
+                        if detected.get('confidence') > 0.8:
+                            return original.decode(detected.get('encoding'))
                     except:
-                        return ek(original, *args)
+                        pass
+
+                    return ek(original, *args)
                 except:
                     raise
     except:

couchpotato/core/media/__init__.py

@@ -1,9 +1,10 @@
 import os
 import traceback
 
-from couchpotato import CPLog
+from couchpotato import CPLog, md5
 from couchpotato.core.event import addEvent, fireEvent, fireEventAsync
 from couchpotato.core.helpers.encoding import toUnicode
+from couchpotato.core.helpers.variable import getExt
 from couchpotato.core.plugins.base import Plugin
 import six
 
@@ -92,7 +93,15 @@ class MediaBase(Plugin):
             if not isinstance(image, (str, unicode)):
                 continue
 
-            if file_type not in existing_files or len(existing_files.get(file_type, [])) == 0:
+            # Check if it has top image
+            filename = '%s.%s' % (md5(image), getExt(image))
+            existing = existing_files.get(file_type, [])
+            has_latest = False
+            for x in existing:
+                if filename in x:
+                    has_latest = True
+
+            if not has_latest or file_type not in existing_files or len(existing_files.get(file_type, [])) == 0:
                 file_path = fireEvent('file.download', url = image, single = True)
                 if file_path:
                     existing_files[file_type] = [toUnicode(file_path)]

couchpotato/core/media/_base/providers/base.py

@@ -94,6 +94,8 @@ class Provider(Plugin):
                 try:
                     data = XMLTree.fromstring(ss(data))
                     return self.getElements(data, item_path)
+                except XMLTree.ParseError:
+                    log.error('Invalid XML returned, check "%s" manually for issues', url)
                 except:
                     log.error('Failed to parsing %s: %s', (self.getName(), traceback.format_exc()))
 

couchpotato/core/media/_base/providers/nzb/newznab.py

@@ -68,8 +68,12 @@ class Base(NZBProvider, RSS):
             if not date:
                 date = self.getTextElement(nzb, 'pubDate')
 
-            nzb_id = self.getTextElement(nzb, 'guid').split('/')[-1:].pop()
             name = self.getTextElement(nzb, 'title')
+            detail_url = self.getTextElement(nzb, 'guid')
+            nzb_id = detail_url.split('/')[-1:].pop()
+
+            if '://' not in detail_url:
+                detail_url = (cleanHost(host['host']) + self.urls['detail']) % tryUrlencode(nzb_id)
 
             if not name:
                 continue
@@ -103,7 +107,7 @@ class Base(NZBProvider, RSS):
                 'age': self.calculateAge(int(time.mktime(parse(date).timetuple()))),
                 'size': int(self.getElement(nzb, 'enclosure').attrib['length']) / 1024 / 1024,
                 'url': ((self.getUrl(host['host']) + self.urls['download']) % tryUrlencode(nzb_id)) + self.getApiExt(host),
-                'detail_url': (cleanHost(host['host']) + self.urls['detail']) % tryUrlencode(nzb_id),
+                'detail_url': detail_url,
                 'content': self.getTextElement(nzb, 'description'),
                 'description': description,
                 'score': host['extra_score'],
@@ -183,7 +187,7 @@ class Base(NZBProvider, RSS):
                 return 'try_next'
 
         try:
-            data = self.urlopen(url, show_error = False)
+            data = self.urlopen(url, show_error = False, headers = {'User-Agent': Env.getIdentifier()})
             self.limits_reached[host] = False
             return data
         except HTTPError as e:

couchpotato/core/media/_base/providers/nzb/omgwtfnzbs.py

@@ -1,13 +1,9 @@
-from urlparse import urlparse, parse_qs
-import time
-
 from couchpotato.core.event import fireEvent
 from couchpotato.core.helpers.encoding import toUnicode, tryUrlencode
 from couchpotato.core.helpers.rss import RSS
 from couchpotato.core.helpers.variable import tryInt
 from couchpotato.core.logger import CPLog
 from couchpotato.core.media._base.providers.nzb.base import NZBProvider
-from dateutil.parser import parse
 
 
 log = CPLog(__name__)
@@ -16,27 +12,19 @@ log = CPLog(__name__)
 class Base(NZBProvider, RSS):
 
     urls = {
-        'search': 'https://rss.omgwtfnzbs.org/rss-search.php?%s',
-        'detail_url': 'https://omgwtfnzbs.org/details.php?id=%s',
+        'search': 'https://api.omgwtfnzbs.org/json/?%s',
     }
 
     http_time_between_calls = 1   # Seconds
 
     cat_ids = [
-        ([15], ['dvdrip']),
+        ([15], ['dvdrip', 'scr', 'r5', 'tc', 'ts', 'cam']),
         ([15, 16], ['brrip']),
         ([16], ['720p', '1080p', 'bd50']),
         ([17], ['dvdr']),
     ]
     cat_backup_id = 'movie'
 
-    def search(self, movie, quality):
-
-        if quality['identifier'] in fireEvent('quality.pre_releases', single = True):
-            return []
-
-        return super(Base, self).search(movie, quality)
-
     def _searchOnTitle(self, title, movie, quality, results):
 
         q = '%s %s' % (title, movie['info']['year'])
@@ -47,22 +35,20 @@ class Base(NZBProvider, RSS):
             'api': self.conf('api_key', default = ''),
         })
 
-        nzbs = self.getRSSData(self.urls['search'] % params)
+        nzbs = self.getJsonData(self.urls['search'] % params)
 
-        for nzb in nzbs:
+        if isinstance(nzbs, list):
+            for nzb in nzbs:
 
-            enclosure = self.getElement(nzb, 'enclosure').attrib
-            nzb_id = parse_qs(urlparse(self.getTextElement(nzb, 'link')).query).get('id')[0]
-
-            results.append({
-                'id': nzb_id,
-                'name': toUnicode(self.getTextElement(nzb, 'title')),
-                'age': self.calculateAge(int(time.mktime(parse(self.getTextElement(nzb, 'pubDate')).timetuple()))),
-                'size': tryInt(enclosure['length']) / 1024 / 1024,
-                'url': enclosure['url'],
-                'detail_url': self.urls['detail_url'] % nzb_id,
-                'description': self.getTextElement(nzb, 'description')
-            })
+                results.append({
+                    'id': nzb.get('nzbid'),
+                    'name': toUnicode(nzb.get('release')),
+                    'age': self.calculateAge(tryInt(nzb.get('usenetage'))),
+                    'size': tryInt(nzb.get('sizebytes')) / 1024 / 1024,
+                    'url': nzb.get('getnzb'),
+                    'detail_url': nzb.get('details'),
+                    'description': nzb.get('weblink')
+                })
 
 
 config = [{

couchpotato/core/media/_base/providers/torrent/hdaccess.py

@@ -0,0 +1,130 @@
+import re
+import traceback
+
+from couchpotato.core.helpers.variable import tryInt, getIdentifier
+from couchpotato.core.logger import CPLog
+from couchpotato.core.media._base.providers.torrent.base import TorrentProvider
+
+
+log = CPLog(__name__)
+
+
+class Base(TorrentProvider):
+
+    urls = {
+        'test': 'https://hdaccess.net/',
+        'detail': 'https://hdaccess.net/details.php?id=%s',
+        'search': 'https://hdaccess.net/searchapi.php?apikey=%s&username=%s&imdbid=%s&internal=%s',
+        'download': 'https://hdaccess.net/grab.php?torrent=%s&apikey=%s',
+    }
+
+    http_time_between_calls = 1  # Seconds
+
+    def _search(self, movie, quality, results):
+        data = self.getJsonData(self.urls['search'] % (self.conf('apikey'), self.conf('username'), getIdentifier(movie), self.conf('internal_only')))
+
+        if data:
+            try:
+                #for result in data[]:
+                for key, result in data.iteritems():
+                    if tryInt(result['total_results']) == 0:
+                        return
+                    torrentscore = self.conf('extra_score')
+                    releasegroup = result['releasegroup']
+                    resolution = result['resolution']
+                    encoding = result['encoding']
+                    freeleech = tryInt(result['freeleech'])
+                    seeders = tryInt(result['seeders'])
+                    torrent_desc = '/ %s / %s / %s / %s seeders' % (releasegroup, resolution, encoding, seeders)
+
+                    if freeleech > 0 and self.conf('prefer_internal'):
+                        torrent_desc += '/ Internal'
+                        torrentscore += 200
+
+                    if seeders == 0:
+                        torrentscore = 0
+
+                    name = result['release_name']
+                    year = tryInt(result['year'])
+
+                    results.append({
+                        'id': tryInt(result['torrentid']),
+                        'name': re.sub('[^A-Za-z0-9\-_ \(\).]+', '', '%s (%s) %s' % (name, year, torrent_desc)),
+                        'url': self.urls['download'] % (result['torrentid'], self.conf('apikey')),
+                        'detail_url': self.urls['detail'] % result['torrentid'],
+                        'size': tryInt(result['size']),
+                        'seeders': tryInt(result['seeders']),
+                        'leechers': tryInt(result['leechers']),
+                        'age': tryInt(result['age']),
+                        'score': torrentscore
+                    })
+            except:
+                log.error('Failed getting results from %s: %s', (self.getName(), traceback.format_exc()))
+config = [{
+    'name': 'hdaccess',
+    'groups': [
+        {
+            'tab': 'searcher',
+            'list': 'torrent_providers',
+            'name': 'HDAccess',
+            'wizard': True,
+            'description': '<a href="https://hdaccess.net">HDAccess</a>',
+            'icon': 'iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABmJLR0QA/wD/AP+gvaeTAAADuUlEQVQ4yz3T209bdQAH8O/vnNNzWno5FIpAKZdSLi23gWMDtumWuSXOyzJj9M1kyIOPS1xiYuKe9GUPezZZnGIiMTqTxS1bdIuYkG2MWKBAKYVszOgKFkrbA+259HfO+fli/PwPHzI+Pg5CCEAI2VcUlEsl1tHdU7P5bGOkWChEaaUCwvHpmkD93POn6bwgCMQGAMYYYwyCruuQnE7SPzjIstvb8l+bm5fXkokJSmlQEkUQAIpSRH5vd0tyum7I/sA1Z5VH2ctmiGWZjHw4McE1NAZtQ9fD25kXt1VN7es7dNjuGRjiJFeVpWo6slsZPhF/Ys/PPeIs2056ff7zIOS5rpU5/viJEwwEnu3Mi18dojjw0aWP6amz57h9RSE/35zinq2nuGjvIQwOj7K2SKeZWkk0auXSSZ+/ZopSy+CbW1pQKpWu6Jr2/qVPPqWRjm6HWi6Tm999g3RyGbndLCqGgVBrO3F7fHykK0YX47NNtGLYlBq/c+H2iD+3k704dHQUDcFmQVXLyP6zhfTqCl45fQYjx17FemoJunoAk1bQFGoVhkdPwNC0ix2dMT+3llodM02rKdo7gN3dHAEhuH/vNgDg3Pl3cPaNt2GZJpYX5lBbFwClBukfGobL5WrayW6NccVCISY4HIQxYts2Q3J5CXOPHuLlo6NoCoXQ2hbG0JFRpJYWcVDIQ5ZlyL5qW5b9hNlWjKsYBgzDgKppMCoGHty7A0orOHbyNNweL+obGnDm9TdhWSYS8Vn4a2shOZ0QJRGSKIHjeGGtWNhjqqpyG+k04k8eozPai9ZwByavf4kfpyZxZGwMfYOHsbwQx34hB5dL4syKweRq/xpXHwzNapqWSSYWMDszzYqFPEaOn4KiKJiZfoCZ6d8Am+GtC++iXCpjaf4P9vefT8HzfKarp3eWRKMxCILwuWXSz977YIK2RTodDoGH1+OG1+tDlbsKkuiAJEngeWBjNUUnv7rucIiOLyzTvMKJTgnVtbVXLctK3L31g+NAUajL5bEptaDpOnTdgGkzVHl9drms0ju3fnJIkphoaQtfbQiFwAcCAY5wnCE5Xff3i8XX4o9nGksH+8zl9hAGZlWMCivkc9z0L3fZ999+LTCGZKi55YJTFHfye3sc6e/vB88LpK6+iWlqSS4WcpcNXZtwOp3B6mo/REmCSSkEgd+qq3vpRkt75Fp9Y1BZWZwnhq4zEovF/u/MATAti4U7umvyu9kR27aikihC9vvTnV2xufVUMu/2uIksy/9tZvgX49fLmAMx3bsAAAAASUVORK5CYII=',
+            'options': [
+                {
+                    'name': 'enabled',
+                    'type': 'enabler',
+                    'default': False,
+                },
+                {
+                    'name': 'username',
+                    'default': '',
+                    'description': 'Enter your site username.',
+                },
+                {
+                    'name': 'apikey',
+                    'default': '',
+                    'label': 'API Key',
+                    'description': 'Enter your site api key. This can be find on <a href="https://hdaccess.net/usercp.php?action=security">Profile Security</a>',
+                },
+                {
+                    'name': 'seed_ratio',
+                    'label': 'Seed ratio',
+                    'type': 'float',
+                    'default': 0,
+                    'description': 'Will not be (re)moved until this seed ratio is met. HDAccess minimum is 1:1.',
+                },
+                {
+                    'name': 'seed_time',
+                    'label': 'Seed time',
+                    'type': 'int',
+                    'default': 0,
+                    'description': 'Will not be (re)moved until this seed time (in hours) is met. HDAccess minimum is 48 hours.',
+                },
+                {
+                    'name': 'prefer_internal',
+                    'advanced': True,
+                    'type': 'bool',
+                    'default': 1,
+                    'description': 'Favors internal releases over non-internal releases.',
+                },
+                {
+                    'name': 'internal_only',
+                    'advanced': True,
+                    'label': 'Internal Only',
+                    'type': 'bool',
+                    'default': False,
+                    'description': 'Only download releases marked as HDAccess internal',
+                },
+                {
+                    'name': 'extra_score',
+                    'advanced': True,
+                    'label': 'Extra Score',
+                    'type': 'int',
+                    'default': 0,
+                    'description': 'Starting score for each release found via this provider.',
+                }
+            ],
+        },
+    ],
+}]

couchpotato/core/media/_base/providers/torrent/iptorrents.py

@@ -14,11 +14,11 @@ log = CPLog(__name__)
 class Base(TorrentProvider):
 
     urls = {
-        'test': 'https://www.iptorrents.com/',
-        'base_url': 'https://www.iptorrents.com',
-        'login': 'https://www.iptorrents.com/torrents/',
-        'login_check': 'https://www.iptorrents.com/inbox.php',
-        'search': 'https://www.iptorrents.com/torrents/?%s%%s&q=%s&qf=ti&p=%%d',
+        'test': 'https://iptorrents.eu/',
+        'base_url': 'https://iptorrents.eu',
+        'login': 'https://iptorrents.eu/torrents/',
+        'login_check': 'https://iptorrents.eu/inbox.php',
+        'search': 'https://iptorrents.eu/torrents/?%s%%s&q=%s&qf=ti&p=%%d',
     }
 
     http_time_between_calls = 1  # Seconds
@@ -120,7 +120,7 @@ config = [{
             'tab': 'searcher',
             'list': 'torrent_providers',
             'name': 'IPTorrents',
-            'description': '<a href="http://www.iptorrents.com">IPTorrents</a>',
+            'description': '<a href="https://iptorrents.eu">IPTorrents</a>',
             'wizard': True,
             'icon': 'iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAABRklEQVR42qWQO0vDUBiG8zeKY3EqQUtNO7g0J6ZJ1+ifKIIFQXAqDYKCyaaYxM3udrZLHdRFhXrZ6liCW6mubfk874EESgqaeOCF7/Y8hEh41aq6yZi2nyZgBGya9XKtZs4No05pAkZV2YbEmyMMsoSxLQeC46wCTdPPY4HruPQyGIhF97qLWsS78Miydn4XdK46NJ9OsQAYBzMIMf8MQ9wtCnTdWCaIDx/u7uljOIQEe0hiIWPamSTLay3+RxOCSPI9+RJAo7Er9r2bnqjBFAqyK+VyK4f5/Cr5ni8OFKVCz49PFI5GdNvvU7ttE1M1zMU+8AMqFksEhrMnQsBDzqmDAwzx2ehRLwT7yyCI+vSC99c3mozH1NxrJgWWtR1BOECfEJSVCm6WCzJGCA7+IWhBsM4zywDPwEp4vCjx2DzBH2ODAfsDb33Ps6dQwJgAAAAASUVORK5CYII=',
             'options': [

couchpotato/core/media/_base/providers/torrent/kickasstorrents.py

@@ -30,7 +30,7 @@ class Base(TorrentMagnetProvider):
     cat_backup_id = None
 
     proxy_list = [
-        'https://kickass.so',
+        'https://kickass.to',
         'http://kickass.pw',
         'http://kickassto.come.in',
         'http://katproxy.ws',

couchpotato/core/media/_base/providers/torrent/torrentbytes.py

@@ -56,7 +56,7 @@ class Base(TorrentProvider):
 
                     full_id = link['href'].replace('details.php?id=', '')
                     torrent_id = full_id[:6]
-                    name = toUnicode(link.contents[0])
+                    name = toUnicode(link.get('title', link.contents[0]).encode('ISO-8859-1')).strip()
 
                     results.append({
                         'id': torrent_id,

couchpotato/core/media/_base/providers/torrent/torrentday.py

@@ -1,3 +1,4 @@
+import re
 from couchpotato.core.helpers.variable import tryInt
 from couchpotato.core.logger import CPLog
 from couchpotato.core.media._base.providers.torrent.base import TorrentProvider
@@ -8,12 +9,12 @@ log = CPLog(__name__)
 class Base(TorrentProvider):
 
     urls = {
-        'test': 'http://www.td.af/',
-        'login': 'http://www.td.af/torrents/',
-        'login_check': 'http://www.torrentday.com/userdetails.php',
-        'detail': 'http://www.td.af/details.php?id=%s',
-        'search': 'http://www.td.af/V3/API/API.php',
-        'download': 'http://www.td.af/download.php/%s/%s',
+        'test': 'https://torrentday.eu/',
+        'login': 'https://torrentday.eu/torrents/',
+        'login_check': 'https://torrentday.eu/userdetails.php',
+        'detail': 'https://torrentday.eu/details.php?id=%s',
+        'search': 'https://torrentday.eu/V3/API/API.php',
+        'download': 'https://torrentday.eu/download.php/%s/%s',
     }
 
     http_time_between_calls = 1  # Seconds
@@ -55,6 +56,10 @@ class Base(TorrentProvider):
         }
 
     def loginSuccess(self, output):
+        often = re.search('You tried too often, please wait .*</div>', output)
+        if often:
+            raise Exception(often.group(0)[:-6].strip())
+
         return 'Password not correct' not in output
 
     def loginCheckSuccess(self, output):
@@ -68,7 +73,7 @@ config = [{
             'tab': 'searcher',
             'list': 'torrent_providers',
             'name': 'TorrentDay',
-            'description': '<a href="http://www.td.af/">TorrentDay</a>',
+            'description': '<a href="https://torrentday.eu/">TorrentDay</a>',
             'wizard': True,
             'icon': 'iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAC5ElEQVQ4y12TXUgUURTH//fO7Di7foeQJH6gEEEIZZllVohfSG/6UA+RSFAQQj74VA8+Bj30lmAlRVSEvZRfhNhaka5ZUG1paKaW39tq5O6Ou+PM3M4o6m6X+XPPzD3zm/+dcy574r515WfIW8CZBM4YAA5Gc/aQC3yd7oXYEONcsISE5dTDh91HS0t7FEWhBUAeN9ynV/d9qJAgE4AECURAcVsGlCCnly26LMA0IQwTa52dje3d3e3hcPi8qqrrMjcVYI3EHCQZlkFOHBwR2QHh2ASAAIJxWGAQEDxjePhs3527XjJwnb37OHBq0T+Tyyjh+9KnEzNJ7nouc1Q/3A3HGsOvnJy+PSUlj81w2Lny9WuJ6+3AmTjD4HOcrdR2dWXLRQePvyaSLfQOPMPC8mC9iHCsOxSyzJCelzdSXlNzD5ujpb25Wbfc/XXJemTXF4+nnCNq+AMLe50uFfEJTiw4GXSFtiHL0SnIq66+p0kSArqO+eH3RdsAv9+f5vW7L7GICq6rmM8XBCAXlBw90rOyxibn5yzfkg/L09M52/jxqdESaIrBXHYZZbB1GX8cEpySxKIB8S5XcOnvqpli1zuwmrTtoLjw5LOK/eeuWsE4JH5IRPaPZKiKigmPp+5pa+u1aEjIMhEgrRkmi9mgxGUhM7LNJSzOzsE3+cOeExovXOjdytE0LV4zqNZUtV0uZzAGoGkhDH/2YHZiErmv4uyWQnZZWc+hoqL3WzlTExN5hhA8IEwkZWZOxwB++30YG/9GkYCPvqAaHAW5uWPROW86OmqCprUR7z1yZDAGQNuCvkoB/baIKUBWMTYymv+gra3eJNvjXu+B562tFyXqTJ6YuHK8rKwvBmC3vR7cOCPQLWFz8LnfXWUrJo9U19BwMyUlJRjTSMJ2ENxUiGxq9KXQfwqYlnWstvbR5aamG9g0uzM8Q4OFt++3NNixQ2NgYmeN03FOTUv7XVpV9aKisvLl1vN/WVhNc/Fi1NEAAAAASUVORK5CYII=',
             'options': [

couchpotato/core/media/_base/providers/torrent/torrentleech.py

@@ -17,7 +17,7 @@ class Base(TorrentProvider):
         'login': 'https://www.torrentleech.org/user/account/login/',
         'login_check': 'https://torrentleech.org/user/messages',
         'detail': 'https://www.torrentleech.org/torrent/%s',
-        'search': 'https://www.torrentleech.org/torrents/browse/index/query/%s/categories/%d',
+        'search': 'https://www.torrentleech.org/torrents/browse/index/query/%s/categories/%s',
         'download': 'https://www.torrentleech.org%s',
     }
 

couchpotato/core/media/_base/providers/torrent/torrentshack.py

@@ -13,12 +13,12 @@ log = CPLog(__name__)
 class Base(TorrentProvider):
 
     urls = {
-        'test': 'http://torrentshack.eu/',
-        'login': 'http://torrentshack.eu/login.php',
-        'login_check': 'http://torrentshack.eu/inbox.php',
-        'detail': 'http://torrentshack.eu/torrent/%s',
-        'search': 'http://torrentshack.eu/torrents.php?action=advanced&searchstr=%s&scene=%s&filter_cat[%d]=1',
-        'download': 'http://torrentshack.eu/%s',
+        'test': 'https://theshack.us.to/',
+        'login': 'https://theshack.us.to/login.php',
+        'login_check': 'https://theshack.us.to/inbox.php',
+        'detail': 'https://theshack.us.to/torrent/%s',
+        'search': 'https://theshack.us.to/torrents.php?action=advanced&searchstr=%s&scene=%s&filter_cat[%d]=1',
+        'download': 'https://theshack.us.to/%s',
     }
 
     http_time_between_calls = 1  # Seconds

couchpotato/core/media/_base/providers/torrent/torrentz.py

@@ -22,12 +22,12 @@ class Base(TorrentMagnetProvider, RSS):
 
     http_time_between_calls = 0
 
-    def _search(self, media, quality, results):
+    def _searchOnTitle(self, title, media, quality, results):
 
         search_url = self.urls['verified_search'] if self.conf('verified_only') else self.urls['search']
 
         # Create search parameters
-        search_params = self.buildUrl(media)
+        search_params = self.buildUrl(title, media, quality)
 
         smin = quality.get('size_min')
         smax = quality.get('size_max')

couchpotato/core/media/_base/providers/torrent/yify.py

@@ -2,27 +2,25 @@ import traceback
 
 from couchpotato.core.helpers.variable import tryInt, getIdentifier
 from couchpotato.core.logger import CPLog
-from couchpotato.core.media._base.providers.torrent.base import TorrentMagnetProvider
+from couchpotato.core.media._base.providers.torrent.base import TorrentProvider
 
 
 log = CPLog(__name__)
 
 
-class Base(TorrentMagnetProvider):
+class Base(TorrentProvider):
 
     urls = {
-        'test': '%s/api',
-        'search': '%s/api/list.json?keywords=%s',
-        'detail': '%s/api/movie.json?id=%s'
+        'test': '%s/api/v2',
+        'search': '%s/api/v2/list_movies.json?limit=50&query_term=%s'
     }
 
     http_time_between_calls = 1  # seconds
 
     proxy_list = [
         'https://yts.re',
-        'http://ytsproxy.come.in',
-        'http://yts.im',
-        'http://yify-torrents.im',
+        'https://yts.wf',
+        'https://yts.im',
     ]
 
     def search(self, movie, quality):
@@ -41,25 +39,28 @@ class Base(TorrentMagnetProvider):
         search_url = self.urls['search'] % (domain, getIdentifier(movie))
 
         data = self.getJsonData(search_url)
+        data = data.get('data')
 
-        if data and data.get('MovieList'):
+        if isinstance(data, dict) and data.get('movies'):
             try:
-                for result in data.get('MovieList'):
+                for result in data.get('movies'):
 
-                    if result['Quality'] and result['Quality'] not in result['MovieTitle']:
-                        title = result['MovieTitle'] + ' BrRip ' + result['Quality']
-                    else:
-                        title = result['MovieTitle'] + ' BrRip'
+                    for release in result.get('torrents', []):
 
-                    results.append({
-                        'id': result['MovieID'],
-                        'name': title,
-                        'url': result['TorrentMagnetUrl'],
-                        'detail_url': self.urls['detail'] % (domain, result['MovieID']),
-                        'size': self.parseSize(result['Size']),
-                        'seeders': tryInt(result['TorrentSeeds']),
-                        'leechers': tryInt(result['TorrentPeers']),
-                    })
+                        if release['quality'] and release['quality'] not in result['title_long']:
+                            title = result['title_long'] + ' BRRip ' + release['quality']
+                        else:
+                            title = result['title_long'] + ' BRRip'
+
+                        results.append({
+                            'id': release['hash'],
+                            'name': title,
+                            'url': release['url'],
+                            'detail_url': result['url'],
+                            'size': self.parseSize(release['size']),
+                            'seeders': tryInt(release['seeds']),
+                            'leechers': tryInt(release['peers']),
+                        })
 
             except:
                 log.error('Failed getting results from %s: %s', (self.getName(), traceback.format_exc()))

couchpotato/core/media/movie/charts/static/charts.js

@@ -44,11 +44,12 @@ var Charts = new Class({
 
 		if( Cookie.read('suggestions_charts_menu_selected') === 'charts'){
 			self.show();
-			self.fireEvent.delay(0, self, 'created');
 		}
 		else
 			self.el.hide();
 
+		self.fireEvent.delay(0, self, 'created');
+
 	},
 
 	fill: function(json){

couchpotato/core/media/movie/providers/automation/crowdai.py

@@ -0,0 +1,89 @@
+import re
+
+from couchpotato.core.helpers.rss import RSS
+from couchpotato.core.helpers.variable import tryInt, splitString
+from couchpotato.core.logger import CPLog
+from couchpotato.core.media.movie.providers.automation.base import Automation
+
+
+log = CPLog(__name__)
+
+autoload = 'CrowdAI'
+
+
+class CrowdAI(Automation, RSS):
+
+    interval = 1800
+
+    def getIMDBids(self):
+
+        movies = []
+
+        urls = dict(zip(splitString(self.conf('automation_urls')), [tryInt(x) for x in splitString(self.conf('automation_urls_use'))]))
+
+        for url in urls:
+
+            if not urls[url]:
+                continue
+
+            rss_movies = self.getRSSData(url)
+
+            for movie in rss_movies:
+
+                description = self.getTextElement(movie, 'description')
+                grabs = 0
+
+                for item in movie:
+                    if item.attrib.get('name') == 'grabs':
+                        grabs = item.attrib.get('value')
+                        break
+
+                if int(grabs) > tryInt(self.conf('number_grabs')):
+                    title = re.match(r'.*Title: .a href.*/">(.*) \(\d{4}\).*', description).group(1)
+                    log.info2('%s grabs for movie: %s, enqueue...', (grabs, title))
+                    year = re.match(r'.*Year: (\d{4}).*', description).group(1)
+                    imdb = self.search(title, year)
+
+                    if imdb and self.isMinimalMovie(imdb):
+                        movies.append(imdb['imdb'])
+
+        return movies
+
+
+config = [{
+    'name': 'crowdai',
+    'groups': [
+        {
+            'tab': 'automation',
+            'list': 'automation_providers',
+            'name': 'crowdai_automation',
+            'label': 'CrowdAI',
+            'description': 'Imports from any newznab powered NZB providers RSS feed depending on the number of grabs per movie. Go to your newznab site and find the RSS section. Then copy the copy paste the link under "Movies > x264 feed" here.',
+            'options': [
+                {
+                    'name': 'automation_enabled',
+                    'default': False,
+                    'type': 'enabler',
+                },
+                {
+                    'name': 'automation_urls_use',
+                    'label': 'Use',
+                    'default': '1',
+                },
+                {
+                    'name': 'automation_urls',
+                    'label': 'url',
+                    'type': 'combined',
+                    'combine': ['automation_urls_use', 'automation_urls'],
+                    'default': 'http://YOUR_PROVIDER/rss?t=THE_MOVIE_CATEGORY&i=YOUR_USER_ID&r=YOUR_API_KEY&res=2&rls=2&num=100',
+                },
+                {
+                    'name': 'number_grabs',
+                    'default': '500',
+                    'label': 'Grab threshold',
+                    'description': 'Number of grabs required',
+                },
+            ],
+        },
+    ],
+}]

couchpotato/core/media/movie/providers/automation/letterboxd.py

@@ -48,11 +48,12 @@ class Letterboxd(Automation):
 
             soup = BeautifulSoup(self.getHTMLData(self.url % username))
 
-            for movie in soup.find_all('a', attrs = {'class': 'frame'}):
-                match = removeEmpty(self.pattern.split(movie['title']))
+            for movie in soup.find_all('li', attrs = {'class': 'poster-container'}):
+                img = movie.find('img', movie)
+                title = img.get('alt')
+
                 movies.append({
-                    'title': match[0],
-                    'year': match[1]
+                    'title': title
                 })
 
         return movies

couchpotato/core/media/movie/providers/automation/rottentomatoes.py

@@ -39,15 +39,14 @@ class Rottentomatoes(Automation, RSS):
 
                 if result:
 
-                    log.info2('Something smells...')
                     rating = tryInt(self.getTextElement(movie, rating_tag))
                     name = result.group(0)
 
+                    print rating, tryInt(self.conf('tomatometer_percent'))
                     if rating < tryInt(self.conf('tomatometer_percent')):
                         log.info2('%s seems to be rotten...', name)
                     else:
-
-                        log.info2('Found %s fresh enough movies, enqueuing: %s', (rating, name))
+                        log.info2('Found %s with fresh rating %s', (name, rating))
                         year = datetime.datetime.now().strftime("%Y")
                         imdb = self.search(name, year)
 

couchpotato/core/media/movie/providers/info/omdbapi.py

@@ -2,6 +2,7 @@ import json
 import re
 import traceback
 
+from couchpotato import Env
 from couchpotato.core.event import addEvent, fireEvent
 from couchpotato.core.helpers.encoding import tryUrlencode
 from couchpotato.core.helpers.variable import tryInt, tryFloat, splitString
@@ -17,8 +18,8 @@ autoload = 'OMDBAPI'
 class OMDBAPI(MovieProvider):
 
     urls = {
-        'search': 'http://www.omdbapi.com/?%s',
-        'info': 'http://www.omdbapi.com/?i=%s',
+        'search': 'http://www.omdbapi.com/?type=movie&%s',
+        'info': 'http://www.omdbapi.com/?type=movie&i=%s',
     }
 
     http_time_between_calls = 0
@@ -38,7 +39,8 @@ class OMDBAPI(MovieProvider):
             }
 
         cache_key = 'omdbapi.cache.%s' % q
-        cached = self.getCache(cache_key, self.urls['search'] % tryUrlencode({'t': name_year.get('name'), 'y': name_year.get('year', '')}), timeout = 3)
+        url = self.urls['search'] % tryUrlencode({'t': name_year.get('name'), 'y': name_year.get('year', '')})
+        cached = self.getCache(cache_key, url, timeout = 3, headers = {'User-Agent': Env.getIdentifier()})
 
         if cached:
             result = self.parseMovie(cached)
@@ -56,7 +58,7 @@ class OMDBAPI(MovieProvider):
             return {}
 
         cache_key = 'omdbapi.cache.%s' % identifier
-        cached = self.getCache(cache_key, self.urls['info'] % identifier, timeout = 3)
+        cached = self.getCache(cache_key, self.urls['info'] % identifier, timeout = 3, headers = {'User-Agent': Env.getIdentifier()})
 
         if cached:
             result = self.parseMovie(cached)

couchpotato/core/media/movie/providers/torrent/bitsoup.py

@@ -11,7 +11,7 @@ autoload = 'Bitsoup'
 class Bitsoup(MovieProvider, Base):
     cat_ids = [
         ([17], ['3d']),
-        ([41], ['720p', '1080p']),
+        ([80], ['720p', '1080p']),
         ([20], ['dvdr']),
         ([19], ['brrip', 'dvdrip']),
     ]

couchpotato/core/media/movie/providers/torrent/hdaccess.py

@@ -0,0 +1,11 @@
+from couchpotato.core.logger import CPLog
+from couchpotato.core.media._base.providers.torrent.hdaccess import Base
+from couchpotato.core.media.movie.providers.base import MovieProvider
+
+log = CPLog(__name__)
+
+autoload = 'HDAccess'
+
+
+class HDAccess(MovieProvider, Base):
+    pass

couchpotato/core/media/movie/providers/torrent/torrentleech.py

@@ -16,12 +16,12 @@ class TorrentLeech(MovieProvider, Base):
         ([9], ['ts', 'tc']),
         ([10], ['r5', 'scr']),
         ([11], ['dvdrip']),
-        ([14], ['brrip']),
+        ([13, 14], ['brrip']),
         ([12], ['dvdr']),
     ]
 
     def buildUrl(self, title, media, quality):
         return (
             tryUrlencode(title.replace(':', '')),
-            self.getCatId(quality)[0]
+            ','.join([str(x) for x in self.getCatId(quality)])
         )

couchpotato/core/media/movie/providers/torrent/torrentshack.py

@@ -22,8 +22,8 @@ class TorrentShack(MovieProvider, Base):
     #   Movies-SD Pack - 983 (not included)
 
     cat_ids = [
-        ([970], ['bd50']),
-        ([300], ['720p', '1080p']),
+        ([970, 320], ['bd50']),
+        ([300, 320], ['720p', '1080p']),
         ([350], ['dvdr']),
         ([400], ['brrip', 'dvdrip']),
     ]

couchpotato/core/media/movie/providers/torrent/torrentz.py

@@ -1,6 +1,5 @@
 from couchpotato.core.helpers.encoding import tryUrlencode
 from couchpotato.core.logger import CPLog
-from couchpotato.core.event import fireEvent
 from couchpotato.core.media._base.providers.torrent.torrentz import Base
 from couchpotato.core.media.movie.providers.base import MovieProvider
 
@@ -11,5 +10,5 @@ autoload = 'Torrentz'
 
 class Torrentz(MovieProvider, Base):
 
-    def buildUrl(self, media):
-        return tryUrlencode('"%s"' % fireEvent('library.query', media, single = True))
+    def buildUrl(self, title, media, quality):
+        return tryUrlencode('"%s %s"' % (title, media['info']['year']))

couchpotato/core/media/movie/suggestion/static/suggest.js

@@ -51,8 +51,8 @@ var SuggestList = new Class({
 			self.show();
 		else
 			self.hide();
-
-		self.fireEvent('created');
+		
+		self.fireEvent.delay(0, self, 'created');
 
 	},
 

couchpotato/core/notifications/core/main.py

@@ -149,16 +149,15 @@ class CoreNotifier(Notification):
     def notify(self, message = '', data = None, listener = None):
         if not data: data = {}
 
+        n = {
+            '_t': 'notification',
+            'time': int(time.time()),
+        }
+
         try:
             db = get_db()
 
-            data['notification_type'] = listener if listener else 'unknown'
-
-            n = {
-                '_t': 'notification',
-                'time': int(time.time()),
-                'message': toUnicode(message)
-            }
+            n['message'] = toUnicode(message)
 
             if data.get('sticky'):
                 n['sticky'] = True
@@ -171,7 +170,7 @@ class CoreNotifier(Notification):
 
             return True
         except:
-            log.error('Failed notify: %s', traceback.format_exc())
+            log.error('Failed notify "%s": %s', (n, traceback.format_exc()))
 
     def frontend(self, type = 'notification', data = None, message = None):
         if not data: data = {}

couchpotato/core/notifications/plex/__init__.py

@@ -23,6 +23,26 @@ config = [{
                     'default': 'localhost',
                     'description': 'Hostname/IP, default localhost'
                 },
+                {
+                    'name': 'username',
+                    'label': 'Username',
+                    'default': '',
+                    'description': 'Required for myPlex'
+                },
+                {
+                    'name': 'password',
+                    'label': 'Password',
+                    'default': '',
+                    'type': 'password',
+                    'description': 'Required for myPlex'
+                },
+                {
+                    'name': 'auth_token',
+                    'label': 'Auth Token',
+                    'default': '',
+                    'advanced': True,
+                    'description': 'Required for myPlex'
+                },
                 {
                     'name': 'clients',
                     'default': '',

couchpotato/core/notifications/plex/server.py

@@ -35,11 +35,46 @@ class PlexServer(object):
         if path.startswith('/'):
             path = path[1:]
 
-        data = self.plex.urlopen('%s/%s' % (
-            self.createHost(self.plex.conf('media_server'), port = 32400),
-            path
-        ))
+        #Maintain support for older Plex installations without myPlex
+        if not self.plex.conf('auth_token') and not self.plex.conf('username') and not self.plex.conf('password'):
+            data = self.plex.urlopen('%s/%s' % (
+                self.createHost(self.plex.conf('media_server'), port = 32400),
+                path
+            ))
+        else:
+            #Fetch X-Plex-Token if it doesn't exist but a username/password do
+            if not self.plex.conf('auth_token') and (self.plex.conf('username') and self.plex.conf('password')):
+                import urllib2, base64
+                log.info("Fetching a new X-Plex-Token from plex.tv")
+                username = self.plex.conf('username')
+                password = self.plex.conf('password')
+                req = urllib2.Request("https://plex.tv/users/sign_in.xml", data="")
+                authheader = "Basic %s" % base64.encodestring('%s:%s' % (username, password))[:-1]
+                req.add_header("Authorization", authheader)
+                req.add_header("X-Plex-Product", "Couchpotato Notifier")
+                req.add_header("X-Plex-Client-Identifier", "b3a6b24dcab2224bdb101fc6aa08ea5e2f3147d6")
+                req.add_header("X-Plex-Version", "1.0")
+                
+                try:
+                    response = urllib2.urlopen(req)
+                except urllib2.URLError, e:
+                    log.info("Error fetching token from plex.tv")
+                
+                try:
+                    auth_tree = etree.parse(response)
+                    token = auth_tree.findall(".//authentication-token")[0].text
+                    self.plex.conf('auth_token', token)
 
+                except (ValueError, IndexError) as e:
+                    log.info("Error parsing plex.tv response: " + ex(e))
+
+            #Add X-Plex-Token header for myPlex support workaround
+            data = self.plex.urlopen('%s/%s?X-Plex-Token=%s' % (
+                self.createHost(self.plex.conf('media_server'), port = 32400),
+                path,
+                self.plex.conf('auth_token')
+            ))
+        
         if data_type == 'xml':
             return etree.fromstring(data)
         else:

couchpotato/core/notifications/webhook.py

@@ -0,0 +1,68 @@
+import traceback
+
+from couchpotato.core.helpers.encoding import toUnicode
+from couchpotato.core.helpers.variable import getIdentifier
+from couchpotato.core.logger import CPLog
+from couchpotato.core.notifications.base import Notification
+
+
+log = CPLog(__name__)
+
+autoload = 'Webhook'
+
+class Webhook(Notification):
+
+    def notify(self, message = '', data = None, listener = None):
+        if not data: data = {}
+
+        post_data = {
+            'message': toUnicode(message)
+        }
+
+        if getIdentifier(data):
+            post_data.update({
+                'imdb_id': getIdentifier(data)
+            })
+
+        headers = {
+            'Content-type': 'application/x-www-form-urlencoded'
+        }
+
+        try:
+            self.urlopen(self.conf('url'), headers = headers, data = post_data, show_error = False)
+            return True
+        except:
+            log.error('Webhook notification failed: %s', traceback.format_exc())
+
+        return False
+
+
+config = [{
+    'name': 'webhook',
+    'groups': [
+        {
+            'tab': 'notifications',
+            'list': 'notification_providers',
+            'name': 'webhook',
+            'label': 'Webhook',
+            'options': [
+                {
+                    'name': 'enabled',
+                    'default': 0,
+                    'type': 'enabler',
+                },
+                {
+                    'name': 'url',
+                    'description': 'The URL to send notification data to when '
+                },
+                {
+                    'name': 'on_snatch',
+                    'default': 0,
+                    'type': 'bool',
+                    'advanced': True,
+                    'description': 'Also send message when movie is snatched.',
+                }
+            ]
+        }
+    ]
+}]

couchpotato/core/plugins/base.py

@@ -39,7 +39,7 @@ class Plugin(object):
 
     _locks = {}
 
-    user_agent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:24.0) Gecko/20130519 Firefox/24.0'
+    user_agent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:34.0) Gecko/20100101 Firefox/34.0'
     http_last_use = {}
     http_time_between_calls = 0
     http_failed_request = {}
@@ -196,7 +196,7 @@ class Plugin(object):
         headers['Host'] = headers.get('Host', None)
         headers['User-Agent'] = headers.get('User-Agent', self.user_agent)
         headers['Accept-encoding'] = headers.get('Accept-encoding', 'gzip')
-        headers['Connection'] = headers.get('Connection', 'close')
+        headers['Connection'] = headers.get('Connection', 'keep-alive')
         headers['Cache-Control'] = headers.get('Cache-Control', 'max-age=0')
 
         r = Env.get('http_opener')
@@ -206,7 +206,7 @@ class Plugin(object):
             if self.http_failed_disabled[host] > (time.time() - 900):
                 log.info2('Disabled calls to %s for 15 minutes because so many failed requests.', host)
                 if not show_error:
-                    raise Exception('Disabled calls to %s for 15 minutes because so many failed requests')
+                    raise Exception('Disabled calls to %s for 15 minutes because so many failed requests' % host)
                 else:
                     return ''
             else:

couchpotato/core/plugins/browser.py

@@ -87,6 +87,7 @@ class FileBrowser(Plugin):
         try:
             dirs = self.getDirectories(path = path, show_hidden = show_hidden)
         except:
+            log.error('Failed getting directory "%s" : %s', (path, traceback.format_exc()))
             dirs = []
 
         parent = os.path.dirname(path.rstrip(os.path.sep))

couchpotato/core/plugins/quality/main.py

@@ -240,7 +240,7 @@ class QualityPlugin(Plugin):
 
         # Add additional size score if only 1 size validated
         if len(size_scores) == 1:
-            self.calcScore(score, size_scores[0], 8)
+            self.calcScore(score, size_scores[0], 7)
         del size_scores
 
         # Return nothing if all scores are <= 0
@@ -491,6 +491,7 @@ class QualityPlugin(Plugin):
             'Movie Name.2014.720p Web-Dl Aac2.0 h264-ReleaseGroup': {'size': 3800, 'quality': 'brrip'},
             'Movie Name.2014.720p.WEBRip.x264.AC3-ReleaseGroup': {'size': 3000, 'quality': 'scr'},
             'Movie.Name.2014.1080p.HDCAM.-.ReleaseGroup': {'size': 5300, 'quality': 'cam'},
+            'Movie.Name.2014.720p.HDSCR.4PARTS.MP4.AAC.ReleaseGroup': {'size': 2401, 'quality': 'scr'},
         }
 
         correct = 0

couchpotato/core/plugins/renamer.py

@@ -885,7 +885,9 @@ Remove it if you want it to be renamed (again, or at least let it try again)
                 #If information is not available, we don't want the tag in the filename
                 replaced = replaced.replace('<' + x + '>', '')
 
-        replaced = self.replaceDoubles(replaced.lstrip('. '))
+        if self.conf('replace_doubles'):
+           replaced = self.replaceDoubles(replaced.lstrip('. '))       
+        
         for x, r in replacements.items():
             if x in ['thename', 'namethe']:
                 replaced = replaced.replace(six.u('<%s>') % toUnicode(x), toUnicode(r))
@@ -1342,6 +1344,14 @@ config = [{
                     'type': 'choice',
                     'options': rename_options
                 },
+                {
+                    'advanced': True,                
+                    'name': 'replace_doubles',
+                    'type': 'bool',
+                    'label': 'Clean Name',
+                    'description': ('Attempt to clean up double separaters due to missing data for fields.','Sometimes this eliminates wanted white space (see <a href="https://github.com/RuudBurger/CouchPotatoServer/issues/2782">#2782</a>).'),
+                    'default': True 
+                },
                 {
                     'name': 'unrar',
                     'type': 'bool',

couchpotato/core/plugins/subtitle.py

@@ -16,7 +16,7 @@ autoload = 'Subtitle'
 
 class Subtitle(Plugin):
 
-    services = ['opensubtitles', 'thesubdb', 'subswiki', 'podnapisi', 'subscenter']
+    services = ['opensubtitles', 'thesubdb', 'subswiki', 'subscenter']
 
     def __init__(self):
         addEvent('renamer.before', self.searchSingle)

couchpotato/static/scripts/page/about.js

@@ -117,7 +117,7 @@ var AboutSettingTab = new Class({
 		var self = this;
 		var date = new Date(json.version.date * 1000);
 		self.version_text.set('text', json.version.hash + (json.version.date ? ' ('+date.toLocaleString()+')' : ''));
-		self.updater_type.set('text', json.version.type + ', ' + json.branch);
+		self.updater_type.set('text', (json.version.type != json.branch) ? (json.version.type + ', ' + json.branch) : json.branch);
 	}
 
 });

init/ubuntu

@@ -95,6 +95,8 @@ fi
 
 case "$1" in
   start)
+        touch $PID_FILE
+        chown $RUN_AS $PID_FILE
         echo "Starting $DESC"
         start-stop-daemon -d $APP_PATH -c $RUN_AS $EXTRA_SSD_OPTS --start --pidfile $PID_FILE --exec $DAEMON -- $DAEMON_OPTS
         ;;

libs/chardet/__init__.py

@@ -15,7 +15,7 @@
 # 02110-1301  USA
 ######################### END LICENSE BLOCK #########################
 
-__version__ = "2.2.1"
+__version__ = "2.3.0"
 from sys import version_info
 
 

libs/chardet/chardetect.py

@@ -12,34 +12,68 @@ Example::
 If no paths are provided, it takes its input from stdin.
 
 """
-from io import open
-from sys import argv, stdin
 
+from __future__ import absolute_import, print_function, unicode_literals
+
+import argparse
+import sys
+from io import open
+
+from chardet import __version__
 from chardet.universaldetector import UniversalDetector
 
 
-def description_of(file, name='stdin'):
-    """Return a string describing the probable encoding of a file."""
+def description_of(lines, name='stdin'):
+    """
+    Return a string describing the probable encoding of a file or
+    list of strings.
+
+    :param lines: The lines to get the encoding of.
+    :type lines: Iterable of bytes
+    :param name: Name of file or collection of lines
+    :type name: str
+    """
     u = UniversalDetector()
-    for line in file:
+    for line in lines:
         u.feed(line)
     u.close()
     result = u.result
     if result['encoding']:
-        return '%s: %s with confidence %s' % (name,
-                                              result['encoding'],
-                                              result['confidence'])
+        return '{0}: {1} with confidence {2}'.format(name, result['encoding'],
+                                                     result['confidence'])
     else:
-        return '%s: no result' % name
+        return '{0}: no result'.format(name)
 
 
-def main():
-    if len(argv) <= 1:
-        print(description_of(stdin))
-    else:
-        for path in argv[1:]:
-            with open(path, 'rb') as f:
-                print(description_of(f, path))
+def main(argv=None):
+    '''
+    Handles command line arguments and gets things started.
+
+    :param argv: List of arguments, as if specified on the command-line.
+                 If None, ``sys.argv[1:]`` is used instead.
+    :type argv: list of str
+    '''
+    # Get command line arguments
+    parser = argparse.ArgumentParser(
+        description="Takes one or more file paths and reports their detected \
+                     encodings",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        conflict_handler='resolve')
+    parser.add_argument('input',
+                        help='File whose encoding we would like to determine.',
+                        type=argparse.FileType('rb'), nargs='*',
+                        default=[sys.stdin])
+    parser.add_argument('--version', action='version',
+                        version='%(prog)s {0}'.format(__version__))
+    args = parser.parse_args(argv)
+
+    for f in args.input:
+        if f.isatty():
+            print("You are running chardetect interactively. Press " +
+                  "CTRL-D twice at the start of a blank line to signal the " +
+                  "end of your input. If you want help, run chardetect " +
+                  "--help\n", file=sys.stderr)
+        print(description_of(f, f.name))
 
 
 if __name__ == '__main__':

libs/chardet/jpcntx.py

@@ -177,6 +177,12 @@ class JapaneseContextAnalysis:
         return -1, 1
 
 class SJISContextAnalysis(JapaneseContextAnalysis):
+    def __init__(self):
+        self.charset_name = "SHIFT_JIS"
+
+    def get_charset_name(self):
+        return self.charset_name
+
     def get_order(self, aBuf):
         if not aBuf:
             return -1, 1
@@ -184,6 +190,8 @@ class SJISContextAnalysis(JapaneseContextAnalysis):
         first_char = wrap_ord(aBuf[0])
         if ((0x81 <= first_char <= 0x9F) or (0xE0 <= first_char <= 0xFC)):
             charLen = 2
+            if (first_char == 0x87) or (0xFA <= first_char <= 0xFC):
+                self.charset_name = "CP932"
         else:
             charLen = 1
 

libs/chardet/latin1prober.py

@@ -129,11 +129,11 @@ class Latin1Prober(CharSetProber):
         if total < 0.01:
             confidence = 0.0
         else:
-            confidence = ((self._mFreqCounter[3] / total)
-                          - (self._mFreqCounter[1] * 20.0 / total))
+            confidence = ((self._mFreqCounter[3] - self._mFreqCounter[1] * 20.0)
+                          / total)
         if confidence < 0.0:
             confidence = 0.0
         # lower the confidence of latin1 so that other more accurate
         # detector can take priority.
-        confidence = confidence * 0.5
+        confidence = confidence * 0.73
         return confidence

libs/chardet/mbcssm.py

@@ -353,7 +353,7 @@ SJIS_cls = (
     2,2,2,2,2,2,2,2,  # 68 - 6f
     2,2,2,2,2,2,2,2,  # 70 - 77
     2,2,2,2,2,2,2,1,  # 78 - 7f
-    3,3,3,3,3,3,3,3,  # 80 - 87
+    3,3,3,3,3,2,2,3,  # 80 - 87
     3,3,3,3,3,3,3,3,  # 88 - 8f
     3,3,3,3,3,3,3,3,  # 90 - 97
     3,3,3,3,3,3,3,3,  # 98 - 9f
@@ -369,9 +369,8 @@ SJIS_cls = (
     2,2,2,2,2,2,2,2,  # d8 - df
     3,3,3,3,3,3,3,3,  # e0 - e7
     3,3,3,3,3,4,4,4,  # e8 - ef
-    4,4,4,4,4,4,4,4,  # f0 - f7
-    4,4,4,4,4,0,0,0   # f8 - ff
-)
+    3,3,3,3,3,3,3,3,  # f0 - f7
+    3,3,3,3,3,0,0,0)  # f8 - ff
 
 
 SJIS_st = (
@@ -571,5 +570,3 @@ UTF8SMModel = {'classTable': UTF8_cls,
                'stateTable': UTF8_st,
                'charLenTable': UTF8CharLenTable,
                'name': 'UTF-8'}
-
-# flake8: noqa

libs/chardet/sjisprober.py

@@ -47,7 +47,7 @@ class SJISProber(MultiByteCharSetProber):
         self._mContextAnalyzer.reset()
 
     def get_charset_name(self):
-        return "SHIFT_JIS"
+        return self._mContextAnalyzer.get_charset_name()
 
     def feed(self, aBuf):
         aLen = len(aBuf)

libs/chardet/universaldetector.py

@@ -71,9 +71,9 @@ class UniversalDetector:
 
         if not self._mGotData:
             # If the data starts with BOM, we know it is UTF
-            if aBuf[:3] == codecs.BOM:
+            if aBuf[:3] == codecs.BOM_UTF8:
                 # EF BB BF  UTF-8 with BOM
-                self.result = {'encoding': "UTF-8", 'confidence': 1.0}
+                self.result = {'encoding': "UTF-8-SIG", 'confidence': 1.0}
             elif aBuf[:4] == codecs.BOM_UTF32_LE:
                 # FF FE 00 00  UTF-32, little-endian BOM
                 self.result = {'encoding': "UTF-32LE", 'confidence': 1.0}

libs/pio/api.py

@@ -154,13 +154,14 @@ class _File(_BaseResource):
         return [cls(f) for f in files]
 
     @classmethod
-    def upload(cls, path, name=None):
+    def upload(cls, path, name=None, parent_id=0):
         with open(path) as f:
             if name:
                 files = {'file': (name, f)}
             else:
                 files = {'file': f}
-            d = cls.client.request('/files/upload', method='POST', files=files)
+            d = cls.client.request('/files/upload', method='POST',
+                                   data={'parent_id': parent_id}, files=files)
 
         f = d['file']
         return cls(f)
@@ -239,7 +240,7 @@ class _Transfer(_BaseResource):
     @classmethod
     def add_url(cls, url, parent_id=0, extract=False, callback_url=None):
         d = cls.client.request('/transfers/add', method='POST', data=dict(
-            url=url, parent_id=parent_id, extract=extract,
+            url=url, save_parent_id=parent_id, extract=extract,
             callback_url=callback_url))
         t = d['transfer']
         return cls(t)
@@ -249,7 +250,7 @@ class _Transfer(_BaseResource):
         with open(path) as f:
             files = {'file': f}
             d = cls.client.request('/files/upload', method='POST', files=files,
-                                   data=dict(parent_id=parent_id,
+                                   data=dict(save_parent_id=parent_id,
                                              extract=extract,
                                              callback_url=callback_url))
         t = d['transfer']

libs/requests/__init__.py

@@ -13,7 +13,7 @@ Requests is an HTTP library, written in Python, for human beings. Basic GET
 usage:
 
    >>> import requests
-   >>> r = requests.get('http://python.org')
+   >>> r = requests.get('https://www.python.org')
    >>> r.status_code
    200
    >>> 'Python is a programming language' in r.content
@@ -22,7 +22,7 @@ usage:
 ... or POST:
 
    >>> payload = dict(key1='value1', key2='value2')
-   >>> r = requests.post("http://httpbin.org/post", data=payload)
+   >>> r = requests.post('http://httpbin.org/post', data=payload)
    >>> print(r.text)
    {
      ...
@@ -42,8 +42,8 @@ is at <http://python-requests.org>.
 """
 
 __title__ = 'requests'
-__version__ = '2.4.0'
-__build__ = 0x020400
+__version__ = '2.5.1'
+__build__ = 0x020501
 __author__ = 'Kenneth Reitz'
 __license__ = 'Apache 2.0'
 __copyright__ = 'Copyright 2014 Kenneth Reitz'

libs/requests/adapters.py

@@ -15,19 +15,21 @@ from .packages.urllib3 import Retry
 from .packages.urllib3.poolmanager import PoolManager, proxy_from_url
 from .packages.urllib3.response import HTTPResponse
 from .packages.urllib3.util import Timeout as TimeoutSauce
-from .compat import urlparse, basestring, urldefrag
+from .compat import urlparse, basestring
 from .utils import (DEFAULT_CA_BUNDLE_PATH, get_encoding_from_headers,
-                    prepend_scheme_if_needed, get_auth_from_url)
+                    prepend_scheme_if_needed, get_auth_from_url, urldefragauth)
 from .structures import CaseInsensitiveDict
 from .packages.urllib3.exceptions import ConnectTimeoutError
 from .packages.urllib3.exceptions import HTTPError as _HTTPError
 from .packages.urllib3.exceptions import MaxRetryError
 from .packages.urllib3.exceptions import ProxyError as _ProxyError
+from .packages.urllib3.exceptions import ProtocolError
 from .packages.urllib3.exceptions import ReadTimeoutError
 from .packages.urllib3.exceptions import SSLError as _SSLError
+from .packages.urllib3.exceptions import ResponseError
 from .cookies import extract_cookies_to_jar
 from .exceptions import (ConnectionError, ConnectTimeout, ReadTimeout, SSLError,
-                         ProxyError)
+                         ProxyError, RetryError)
 from .auth import _basic_auth_str
 
 DEFAULT_POOLBLOCK = False
@@ -59,8 +61,12 @@ class HTTPAdapter(BaseAdapter):
     :param pool_connections: The number of urllib3 connection pools to cache.
     :param pool_maxsize: The maximum number of connections to save in the pool.
     :param int max_retries: The maximum number of retries each connection
-        should attempt. Note, this applies only to failed connections and
-        timeouts, never to requests where the server returns a response.
+        should attempt. Note, this applies only to failed DNS lookups, socket
+        connections and connection timeouts, never to requests where data has
+        made it to the server. By default, Requests does not retry failed
+        connections. If you need granular control over the conditions under
+        which we retry a request, import urllib3's ``Retry`` class and pass
+        that instead.
     :param pool_block: Whether the connection pool should block for connections.
 
     Usage::
@@ -76,7 +82,10 @@ class HTTPAdapter(BaseAdapter):
     def __init__(self, pool_connections=DEFAULT_POOLSIZE,
                  pool_maxsize=DEFAULT_POOLSIZE, max_retries=DEFAULT_RETRIES,
                  pool_block=DEFAULT_POOLBLOCK):
-        self.max_retries = max_retries
+        if max_retries == DEFAULT_RETRIES:
+            self.max_retries = Retry(0, read=False)
+        else:
+            self.max_retries = Retry.from_int(max_retries)
         self.config = {}
         self.proxy_manager = {}
 
@@ -122,7 +131,7 @@ class HTTPAdapter(BaseAdapter):
         self._pool_block = block
 
         self.poolmanager = PoolManager(num_pools=connections, maxsize=maxsize,
-                                       block=block, **pool_kwargs)
+                                       block=block, strict=True, **pool_kwargs)
 
     def proxy_manager_for(self, proxy, **proxy_kwargs):
         """Return urllib3 ProxyManager for the given proxy.
@@ -269,7 +278,7 @@ class HTTPAdapter(BaseAdapter):
         proxy = proxies.get(scheme)
 
         if proxy and scheme != 'https':
-            url, _ = urldefrag(request.url)
+            url = urldefragauth(request.url)
         else:
             url = request.path_url
 
@@ -316,8 +325,10 @@ class HTTPAdapter(BaseAdapter):
 
         :param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
         :param stream: (optional) Whether to stream the request content.
-        :param timeout: (optional) The timeout on the request.
-        :type timeout: float or tuple (connect timeout, read timeout), eg (3.1, 20)
+        :param timeout: (optional) How long to wait for the server to send
+            data before giving up, as a float, or a (`connect timeout, read
+            timeout <user/advanced.html#timeouts>`_) tuple.
+        :type timeout: float or tuple
         :param verify: (optional) Whether to verify SSL certificates.
         :param cert: (optional) Any user-provided SSL certificate to be trusted.
         :param proxies: (optional) The proxies dictionary to apply to the request.
@@ -355,7 +366,7 @@ class HTTPAdapter(BaseAdapter):
                     assert_same_host=False,
                     preload_content=False,
                     decode_content=False,
-                    retries=Retry(self.max_retries, read=False),
+                    retries=self.max_retries,
                     timeout=timeout
                 )
 
@@ -400,13 +411,16 @@ class HTTPAdapter(BaseAdapter):
                     # All is well, return the connection to the pool.
                     conn._put_conn(low_conn)
 
-        except socket.error as sockerr:
-            raise ConnectionError(sockerr, request=request)
+        except (ProtocolError, socket.error) as err:
+            raise ConnectionError(err, request=request)
 
         except MaxRetryError as e:
             if isinstance(e.reason, ConnectTimeoutError):
                 raise ConnectTimeout(e, request=request)
 
+            if isinstance(e.reason, ResponseError):
+                raise RetryError(e, request=request)
+
             raise ConnectionError(e, request=request)
 
         except _ProxyError as e:

libs/requests/api.py

@@ -22,12 +22,17 @@ def request(method, url, **kwargs):
     :param url: URL for the new :class:`Request` object.
     :param params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`.
     :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`.
+    :param json: (optional) json data to send in the body of the :class:`Request`.
     :param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`.
     :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`.
-    :param files: (optional) Dictionary of 'name': file-like-objects (or {'name': ('filename', fileobj)}) for multipart encoding upload.
+    :param files: (optional) Dictionary of ``'name': file-like-objects`` (or ``{'name': ('filename', fileobj)}``) for multipart encoding upload.
     :param auth: (optional) Auth tuple to enable Basic/Digest/Custom HTTP Auth.
-    :param timeout: (optional) Float describing the timeout of the request in seconds.
+    :param timeout: (optional) How long to wait for the server to send data
+        before giving up, as a float, or a (`connect timeout, read timeout
+        <user/advanced.html#timeouts>`_) tuple.
+    :type timeout: float or tuple
     :param allow_redirects: (optional) Boolean. Set to True if POST/PUT/DELETE redirect following is allowed.
+    :type allow_redirects: bool
     :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy.
     :param verify: (optional) if ``True``, the SSL cert will be verified. A CA_BUNDLE path can also be provided.
     :param stream: (optional) if ``False``, the response content will be immediately downloaded.
@@ -41,7 +46,12 @@ def request(method, url, **kwargs):
     """
 
     session = sessions.Session()
-    return session.request(method=method, url=url, **kwargs)
+    response = session.request(method=method, url=url, **kwargs)
+    # By explicitly closing the session, we avoid leaving sockets open which
+    # can trigger a ResourceWarning in some cases, and look like a memory leak
+    # in others.
+    session.close()
+    return response
 
 
 def get(url, **kwargs):
@@ -77,15 +87,16 @@ def head(url, **kwargs):
     return request('head', url, **kwargs)
 
 
-def post(url, data=None, **kwargs):
+def post(url, data=None, json=None, **kwargs):
     """Sends a POST request. Returns :class:`Response` object.
 
     :param url: URL for the new :class:`Request` object.
     :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`.
+    :param json: (optional) json data to send in the body of the :class:`Request`.
     :param \*\*kwargs: Optional arguments that ``request`` takes.
     """
 
-    return request('post', url, data=data, **kwargs)
+    return request('post', url, data=data, json=json, **kwargs)
 
 
 def put(url, data=None, **kwargs):

libs/requests/auth.py

@@ -17,6 +17,7 @@ from base64 import b64encode
 from .compat import urlparse, str
 from .cookies import extract_cookies_to_jar
 from .utils import parse_dict_header, to_native_string
+from .status_codes import codes
 
 CONTENT_TYPE_FORM_URLENCODED = 'application/x-www-form-urlencoded'
 CONTENT_TYPE_MULTI_PART = 'multipart/form-data'
@@ -66,6 +67,7 @@ class HTTPDigestAuth(AuthBase):
         self.nonce_count = 0
         self.chal = {}
         self.pos = None
+        self.num_401_calls = 1
 
     def build_digest_header(self, method, url):
 
@@ -150,6 +152,11 @@ class HTTPDigestAuth(AuthBase):
 
         return 'Digest %s' % (base)
 
+    def handle_redirect(self, r, **kwargs):
+        """Reset num_401_calls counter on redirects."""
+        if r.is_redirect:
+            self.num_401_calls = 1
+
     def handle_401(self, r, **kwargs):
         """Takes the given response and tries digest-auth, if needed."""
 
@@ -162,7 +169,7 @@ class HTTPDigestAuth(AuthBase):
 
         if 'digest' in s_auth.lower() and num_401_calls < 2:
 
-            setattr(self, 'num_401_calls', num_401_calls + 1)
+            self.num_401_calls += 1
             pat = re.compile(r'digest ', flags=re.IGNORECASE)
             self.chal = parse_dict_header(pat.sub('', s_auth, count=1))
 
@@ -182,7 +189,7 @@ class HTTPDigestAuth(AuthBase):
 
             return _r
 
-        setattr(self, 'num_401_calls', 1)
+        self.num_401_calls = 1
         return r
 
     def __call__(self, r):
@@ -192,6 +199,11 @@ class HTTPDigestAuth(AuthBase):
         try:
             self.pos = r.body.tell()
         except AttributeError:
-            pass
+            # In the case of HTTPDigestAuth being reused and the body of
+            # the previous request was a file-like object, pos has the
+            # file position of the previous body. Ensure it's set to
+            # None.
+            self.pos = None
         r.register_hook('response', self.handle_401)
+        r.register_hook('response', self.handle_redirect)
         return r

libs/requests/compat.py

@@ -76,7 +76,7 @@ is_solaris = ('solar==' in str(sys.platform).lower())   # Complete guess.
 try:
     import simplejson as json
 except (ImportError, SyntaxError):
-    # simplejson does not support Python 3.2, it thows a SyntaxError
+    # simplejson does not support Python 3.2, it throws a SyntaxError
     # because of u'...' Unicode literals.
     import json
 

libs/requests/exceptions.py

@@ -46,15 +46,16 @@ class SSLError(ConnectionError):
 class Timeout(RequestException):
     """The request timed out.
 
-    Catching this error will catch both :exc:`ConnectTimeout` and
-    :exc:`ReadTimeout` errors.
+    Catching this error will catch both
+    :exc:`~requests.exceptions.ConnectTimeout` and
+    :exc:`~requests.exceptions.ReadTimeout` errors.
     """
 
 
 class ConnectTimeout(ConnectionError, Timeout):
-    """The request timed out while trying to connect to the server.
+    """The request timed out while trying to connect to the remote server.
 
-    Requests that produce this error are safe to retry
+    Requests that produced this error are safe to retry.
     """
 
 
@@ -88,3 +89,11 @@ class ChunkedEncodingError(RequestException):
 
 class ContentDecodingError(RequestException, BaseHTTPError):
     """Failed to decode response content"""
+
+
+class StreamConsumedError(RequestException, TypeError):
+    """The content for this response was already consumed"""
+
+
+class RetryError(RequestException):
+    """Custom retries logic failed"""

libs/requests/models.py

@@ -20,10 +20,10 @@ from .packages.urllib3.fields import RequestField
 from .packages.urllib3.filepost import encode_multipart_formdata
 from .packages.urllib3.util import parse_url
 from .packages.urllib3.exceptions import (
-    DecodeError, ReadTimeoutError, ProtocolError)
+    DecodeError, ReadTimeoutError, ProtocolError, LocationParseError)
 from .exceptions import (
-    HTTPError, RequestException, MissingSchema, InvalidURL,
-    ChunkedEncodingError, ContentDecodingError, ConnectionError)
+    HTTPError, MissingSchema, InvalidURL, ChunkedEncodingError,
+    ContentDecodingError, ConnectionError, StreamConsumedError)
 from .utils import (
     guess_filename, get_auth_from_url, requote_uri,
     stream_decode_response_unicode, to_key_val_list, parse_header_links,
@@ -46,6 +46,8 @@ DEFAULT_REDIRECT_LIMIT = 30
 CONTENT_CHUNK_SIZE = 10 * 1024
 ITER_CHUNK_SIZE = 512
 
+json_dumps = json.dumps
+
 
 class RequestEncodingMixin(object):
     @property
@@ -189,7 +191,8 @@ class Request(RequestHooksMixin):
     :param url: URL to send.
     :param headers: dictionary of headers to send.
     :param files: dictionary of {filename: fileobject} files to multipart upload.
-    :param data: the body to attach the request. If a dictionary is provided, form-encoding will take place.
+    :param data: the body to attach to the request. If a dictionary is provided, form-encoding will take place.
+    :param json: json for the body to attach to the request (if data is not specified).
     :param params: dictionary of URL parameters to append to the URL.
     :param auth: Auth handler or (user, pass) tuple.
     :param cookies: dictionary or CookieJar of cookies to attach to this request.
@@ -212,7 +215,8 @@ class Request(RequestHooksMixin):
         params=None,
         auth=None,
         cookies=None,
-        hooks=None):
+        hooks=None,
+        json=None):
 
         # Default empty dicts for dict params.
         data = [] if data is None else data
@@ -230,6 +234,7 @@ class Request(RequestHooksMixin):
         self.headers = headers
         self.files = files
         self.data = data
+        self.json = json
         self.params = params
         self.auth = auth
         self.cookies = cookies
@@ -246,6 +251,7 @@ class Request(RequestHooksMixin):
             headers=self.headers,
             files=self.files,
             data=self.data,
+            json=self.json,
             params=self.params,
             auth=self.auth,
             cookies=self.cookies,
@@ -289,14 +295,15 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
         self.hooks = default_hooks()
 
     def prepare(self, method=None, url=None, headers=None, files=None,
-                data=None, params=None, auth=None, cookies=None, hooks=None):
+                data=None, params=None, auth=None, cookies=None, hooks=None,
+                json=None):
         """Prepares the entire request with the given parameters."""
 
         self.prepare_method(method)
         self.prepare_url(url, params)
         self.prepare_headers(headers)
         self.prepare_cookies(cookies)
-        self.prepare_body(data, files)
+        self.prepare_body(data, files, json)
         self.prepare_auth(auth, url)
         # Note that prepare_auth must be last to enable authentication schemes
         # such as OAuth to work on a fully prepared request.
@@ -326,21 +333,27 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
     def prepare_url(self, url, params):
         """Prepares the given HTTP URL."""
         #: Accept objects that have string representations.
-        try:
-            url = unicode(url)
-        except NameError:
-            # We're on Python 3.
-            url = str(url)
-        except UnicodeDecodeError:
-            pass
+        #: We're unable to blindy call unicode/str functions
+        #: as this will include the bytestring indicator (b'')
+        #: on python 3.x.
+        #: https://github.com/kennethreitz/requests/pull/2238
+        if isinstance(url, bytes):
+            url = url.decode('utf8')
+        else:
+            url = unicode(url) if is_py2 else str(url)
 
-        # Don't do any URL preparation for oddball schemes
+        # Don't do any URL preparation for non-HTTP schemes like `mailto`,
+        # `data` etc to work around exceptions from `url_parse`, which
+        # handles RFC 3986 only.
         if ':' in url and not url.lower().startswith('http'):
             self.url = url
             return
 
         # Support for unicode domain names and paths.
-        scheme, auth, host, port, path, query, fragment = parse_url(url)
+        try:
+            scheme, auth, host, port, path, query, fragment = parse_url(url)
+        except LocationParseError as e:
+            raise InvalidURL(*e.args)
 
         if not scheme:
             raise MissingSchema("Invalid URL {0!r}: No schema supplied. "
@@ -397,7 +410,7 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
         else:
             self.headers = CaseInsensitiveDict()
 
-    def prepare_body(self, data, files):
+    def prepare_body(self, data, files, json=None):
         """Prepares the given HTTP body data."""
 
         # Check if file, fo, generator, iterator.
@@ -408,6 +421,10 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
         content_type = None
         length = None
 
+        if json is not None:
+            content_type = 'application/json'
+            body = json_dumps(json)
+
         is_stream = all([
             hasattr(data, '__iter__'),
             not isinstance(data, (basestring, list, tuple, dict))
@@ -433,7 +450,7 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
             if files:
                 (body, content_type) = self._encode_files(files, data)
             else:
-                if data:
+                if data and json is None:
                     body = self._encode_params(data)
                     if isinstance(data, basestring) or hasattr(data, 'read'):
                         content_type = None
@@ -443,7 +460,7 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
             self.prepare_content_length(body)
 
             # Add content-type if it wasn't explicitly provided.
-            if (content_type) and (not 'content-type' in self.headers):
+            if content_type and ('content-type' not in self.headers):
                 self.headers['Content-Type'] = content_type
 
         self.body = body
@@ -457,7 +474,7 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
             l = super_len(body)
             if l:
                 self.headers['Content-Length'] = builtin_str(l)
-        elif self.method not in ('GET', 'HEAD'):
+        elif (self.method not in ('GET', 'HEAD')) and (self.headers.get('Content-Length') is None):
             self.headers['Content-Length'] = '0'
 
     def prepare_auth(self, auth, url=''):
@@ -600,7 +617,7 @@ class Response(object):
     def ok(self):
         try:
             self.raise_for_status()
-        except RequestException:
+        except HTTPError:
             return False
         return True
 
@@ -653,6 +670,8 @@ class Response(object):
 
             self._content_consumed = True
 
+        if self._content_consumed and isinstance(self._content, bool):
+            raise StreamConsumedError()
         # simulate reading small chunks of the content
         reused_chunks = iter_slices(self._content, chunk_size)
 
@@ -665,7 +684,7 @@ class Response(object):
 
         return chunks
 
-    def iter_lines(self, chunk_size=ITER_CHUNK_SIZE, decode_unicode=None):
+    def iter_lines(self, chunk_size=ITER_CHUNK_SIZE, decode_unicode=None, delimiter=None):
         """Iterates over the response data, one line at a time.  When
         stream=True is set on the request, this avoids reading the
         content at once into memory for large responses.
@@ -677,7 +696,11 @@ class Response(object):
 
             if pending is not None:
                 chunk = pending + chunk
-            lines = chunk.splitlines()
+
+            if delimiter:
+                lines = chunk.split(delimiter)
+            else:
+                lines = chunk.splitlines()
 
             if lines and lines[-1] and chunk and lines[-1][-1] == chunk[-1]:
                 pending = lines.pop()

libs/requests/packages/urllib3/__init__.py

@@ -57,7 +57,7 @@ del NullHandler
 
 # Set security warning to only go off once by default.
 import warnings
-warnings.simplefilter('module', exceptions.SecurityWarning)
+warnings.simplefilter('always', exceptions.SecurityWarning)
 
 def disable_warnings(category=exceptions.HTTPWarning):
     """

libs/requests/packages/urllib3/_collections.py

@@ -14,7 +14,7 @@ try: # Python 2.7+
     from collections import OrderedDict
 except ImportError:
     from .packages.ordered_dict import OrderedDict
-from .packages.six import itervalues
+from .packages.six import iterkeys, itervalues
 
 
 __all__ = ['RecentlyUsedContainer', 'HTTPHeaderDict']
@@ -85,8 +85,7 @@ class RecentlyUsedContainer(MutableMapping):
     def clear(self):
         with self.lock:
             # Copy pointers to all values, then wipe the mapping
-            # under Python 2, this copies the list of values twice :-|
-            values = list(self._container.values())
+            values = list(itervalues(self._container))
             self._container.clear()
 
         if self.dispose_func:
@@ -95,7 +94,7 @@ class RecentlyUsedContainer(MutableMapping):
 
     def keys(self):
         with self.lock:
-            return self._container.keys()
+            return list(iterkeys(self._container))
 
 
 class HTTPHeaderDict(MutableMapping):

libs/requests/packages/urllib3/connection.py

@@ -3,6 +3,7 @@ import sys
 import socket
 from socket import timeout as SocketTimeout
 import warnings
+from .packages import six
 
 try:  # Python 3
     from http.client import HTTPConnection as _HTTPConnection, HTTPException
@@ -26,12 +27,20 @@ except (ImportError, AttributeError):  # Platform-specific: No SSL.
         pass
 
 
+try:  # Python 3:
+    # Not a no-op, we're adding this to the namespace so it can be imported.
+    ConnectionError = ConnectionError
+except NameError:  # Python 2:
+    class ConnectionError(Exception):
+        pass
+
+
 from .exceptions import (
     ConnectTimeoutError,
     SystemTimeWarning,
+    SecurityWarning,
 )
 from .packages.ssl_match_hostname import match_hostname
-from .packages import six
 
 from .util.ssl_ import (
     resolve_cert_reqs,
@@ -40,8 +49,8 @@ from .util.ssl_ import (
     assert_fingerprint,
 )
 
-from .util import connection
 
+from .util import connection
 
 port_by_scheme = {
     'http': 80,
@@ -233,8 +242,15 @@ class VerifiedHTTPSConnection(HTTPSConnection):
                                self.assert_fingerprint)
         elif resolved_cert_reqs != ssl.CERT_NONE \
                 and self.assert_hostname is not False:
-            match_hostname(self.sock.getpeercert(),
-                           self.assert_hostname or hostname)
+            cert = self.sock.getpeercert()
+            if not cert.get('subjectAltName', ()):
+                warnings.warn((
+                    'Certificate has no `subjectAltName`, falling back to check for a `commonName` for now. '
+                    'This feature is being removed by major browsers and deprecated by RFC 2818. '
+                    '(See https://github.com/shazow/urllib3/issues/497 for details.)'),
+                    SecurityWarning
+                )
+            match_hostname(cert, self.assert_hostname or hostname)
 
         self.is_verified = (resolved_cert_reqs == ssl.CERT_REQUIRED
                             or self.assert_fingerprint is not None)

libs/requests/packages/urllib3/connectionpool.py

@@ -32,7 +32,7 @@ from .connection import (
     port_by_scheme,
     DummyConnection,
     HTTPConnection, HTTPSConnection, VerifiedHTTPSConnection,
-    HTTPException, BaseSSLError,
+    HTTPException, BaseSSLError, ConnectionError
 )
 from .request import RequestMethods
 from .response import HTTPResponse
@@ -278,6 +278,23 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
             # can be removed later
             return Timeout.from_float(timeout)
 
+    def _raise_timeout(self, err, url, timeout_value):
+        """Is the error actually a timeout? Will raise a ReadTimeout or pass"""
+
+        if isinstance(err, SocketTimeout):
+            raise ReadTimeoutError(self, url, "Read timed out. (read timeout=%s)" % timeout_value)
+
+        # See the above comment about EAGAIN in Python 3. In Python 2 we have
+        # to specifically catch it and throw the timeout error
+        if hasattr(err, 'errno') and err.errno in _blocking_errnos:
+            raise ReadTimeoutError(self, url, "Read timed out. (read timeout=%s)" % timeout_value)
+
+        # Catch possible read timeouts thrown as SSL errors. If not the
+        # case, rethrow the original. We need to do this because of:
+        # http://bugs.python.org/issue10272
+        if 'timed out' in str(err) or 'did not complete (read)' in str(err):  # Python 2.6
+            raise ReadTimeoutError(self, url, "Read timed out. (read timeout=%s)" % timeout_value)
+
     def _make_request(self, conn, method, url, timeout=_Default,
                       **httplib_request_kw):
         """
@@ -301,7 +318,12 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
         conn.timeout = timeout_obj.connect_timeout
 
         # Trigger any extra validation we need to do.
-        self._validate_conn(conn)
+        try:
+            self._validate_conn(conn)
+        except (SocketTimeout, BaseSSLError) as e:
+            # Py2 raises this as a BaseSSLError, Py3 raises it as socket timeout.
+            self._raise_timeout(err=e, url=url, timeout_value=conn.timeout)
+            raise
 
         # conn.request() calls httplib.*.request, not the method in
         # urllib3.request. It also calls makefile (recv) on the socket.
@@ -331,28 +353,8 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
                 httplib_response = conn.getresponse(buffering=True)
             except TypeError:  # Python 2.6 and older
                 httplib_response = conn.getresponse()
-        except SocketTimeout:
-            raise ReadTimeoutError(
-                self, url, "Read timed out. (read timeout=%s)" % read_timeout)
-
-        except BaseSSLError as e:
-            # Catch possible read timeouts thrown as SSL errors. If not the
-            # case, rethrow the original. We need to do this because of:
-            # http://bugs.python.org/issue10272
-            if 'timed out' in str(e) or \
-               'did not complete (read)' in str(e):  # Python 2.6
-                raise ReadTimeoutError(
-                        self, url, "Read timed out. (read timeout=%s)" % read_timeout)
-
-            raise
-
-        except SocketError as e:  # Platform-specific: Python 2
-            # See the above comment about EAGAIN in Python 3. In Python 2 we
-            # have to specifically catch it and throw the timeout error
-            if e.errno in _blocking_errnos:
-                raise ReadTimeoutError(
-                    self, url, "Read timed out. (read timeout=%s)" % read_timeout)
-
+        except (SocketTimeout, BaseSSLError, SocketError) as e:
+            self._raise_timeout(err=e, url=url, timeout_value=read_timeout)
             raise
 
         # AppEngine doesn't have a version attr.
@@ -537,12 +539,15 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
             raise EmptyPoolError(self, "No pool connections are available.")
 
         except (BaseSSLError, CertificateError) as e:
-            # Release connection unconditionally because there is no way to
-            # close it externally in case of exception.
-            release_conn = True
+            # Close the connection. If a connection is reused on which there
+            # was a Certificate error, the next request will certainly raise
+            # another Certificate error.
+            if conn:
+                conn.close()
+                conn = None
             raise SSLError(e)
 
-        except (TimeoutError, HTTPException, SocketError) as e:
+        except (TimeoutError, HTTPException, SocketError, ConnectionError) as e:
             if conn:
                 # Discard the connection for these exceptions. It will be
                 # be replaced during the next _get_conn() call.
@@ -725,8 +730,7 @@ class HTTPSConnectionPool(HTTPConnectionPool):
             warnings.warn((
                 'Unverified HTTPS request is being made. '
                 'Adding certificate verification is strongly advised. See: '
-                'https://urllib3.readthedocs.org/en/latest/security.html '
-                '(This warning will only appear once by default.)'),
+                'https://urllib3.readthedocs.org/en/latest/security.html'),
                 InsecureRequestWarning)
 
 

libs/requests/packages/urllib3/contrib/pyopenssl.py

@@ -29,7 +29,7 @@ Now you can use :mod:`urllib3` as you normally would, and it will support SNI
 when the required modules are installed.
 
 Activating this module also has the positive side effect of disabling SSL/TLS
-encryption in Python 2 (see `CRIME attack`_).
+compression in Python 2 (see `CRIME attack`_).
 
 If you want to configure the default list of supported cipher suites, you can
 set the ``urllib3.contrib.pyopenssl.DEFAULT_SSL_CIPHER_LIST`` variable.
@@ -70,9 +70,14 @@ HAS_SNI = SUBJ_ALT_NAME_SUPPORT
 # Map from urllib3 to PyOpenSSL compatible parameter-values.
 _openssl_versions = {
     ssl.PROTOCOL_SSLv23: OpenSSL.SSL.SSLv23_METHOD,
-    ssl.PROTOCOL_SSLv3: OpenSSL.SSL.SSLv3_METHOD,
     ssl.PROTOCOL_TLSv1: OpenSSL.SSL.TLSv1_METHOD,
 }
+
+try:
+    _openssl_versions.update({ssl.PROTOCOL_SSLv3: OpenSSL.SSL.SSLv3_METHOD})
+except AttributeError:
+    pass
+
 _openssl_verify = {
     ssl.CERT_NONE: OpenSSL.SSL.VERIFY_NONE,
     ssl.CERT_OPTIONAL: OpenSSL.SSL.VERIFY_PEER,
@@ -199,8 +204,21 @@ class WrappedSocket(object):
     def settimeout(self, timeout):
         return self.socket.settimeout(timeout)
 
+    def _send_until_done(self, data):
+        while True:
+            try:
+                return self.connection.send(data)
+            except OpenSSL.SSL.WantWriteError:
+                _, wlist, _ = select.select([], [self.socket], [],
+                                            self.socket.gettimeout())
+                if not wlist:
+                    raise timeout()
+                continue
+
     def sendall(self, data):
-        return self.connection.sendall(data)
+        while len(data):
+            sent = self._send_until_done(data)
+            data = data[sent:]
 
     def close(self):
         if self._makefile_refs < 1:
@@ -248,6 +266,7 @@ def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None,
                     ssl_version=None):
     ctx = OpenSSL.SSL.Context(_openssl_versions[ssl_version])
     if certfile:
+        keyfile = keyfile or certfile  # Match behaviour of the normal python ssl library
         ctx.use_certificate_file(certfile)
     if keyfile:
         ctx.use_privatekey_file(keyfile)

libs/requests/packages/urllib3/exceptions.py

@@ -72,11 +72,8 @@ class MaxRetryError(RequestError):
     def __init__(self, pool, url, reason=None):
         self.reason = reason
 
-        message = "Max retries exceeded with url: %s" % url
-        if reason:
-            message += " (Caused by %r)" % reason
-        else:
-            message += " (Caused by redirect)"
+        message = "Max retries exceeded with url: %s (Caused by %r)" % (
+            url, reason)
 
         RequestError.__init__(self, pool, url, message)
 
@@ -141,6 +138,12 @@ class LocationParseError(LocationValueError):
         self.location = location
 
 
+class ResponseError(HTTPError):
+    "Used as a container for an error reason supplied in a MaxRetryError."
+    GENERIC_ERROR = 'too many error responses'
+    SPECIFIC_ERROR = 'too many {status_code} error responses'
+
+
 class SecurityWarning(HTTPWarning):
     "Warned when perfoming security reducing actions"
     pass

libs/requests/packages/urllib3/request.py

@@ -118,18 +118,24 @@ class RequestMethods(object):
         which is used to compose the body of the request. The random boundary
         string can be explicitly set with the ``multipart_boundary`` parameter.
         """
-        if encode_multipart:
-            body, content_type = encode_multipart_formdata(
-                fields or {}, boundary=multipart_boundary)
-        else:
-            body, content_type = (urlencode(fields or {}),
-                                  'application/x-www-form-urlencoded')
-
         if headers is None:
             headers = self.headers
 
-        headers_ = {'Content-Type': content_type}
-        headers_.update(headers)
+        extra_kw = {'headers': {}}
 
-        return self.urlopen(method, url, body=body, headers=headers_,
-                            **urlopen_kw)
+        if fields:
+            if 'body' in urlopen_kw:
+                raise TypeError('request got values for both \'fields\' and \'body\', can only specify one.')
+
+            if encode_multipart:
+                body, content_type = encode_multipart_formdata(fields, boundary=multipart_boundary)
+            else:
+                body, content_type = urlencode(fields), 'application/x-www-form-urlencoded'
+
+            extra_kw['body'] = body
+            extra_kw['headers'] = {'Content-Type': content_type}
+
+        extra_kw['headers'].update(headers)
+        extra_kw.update(urlopen_kw)
+
+        return self.urlopen(method, url, **extra_kw)

libs/requests/packages/urllib3/util/retry.py

@@ -2,10 +2,11 @@ import time
 import logging
 
 from ..exceptions import (
-    ProtocolError,
     ConnectTimeoutError,
-    ReadTimeoutError,
     MaxRetryError,
+    ProtocolError,
+    ReadTimeoutError,
+    ResponseError,
 )
 from ..packages import six
 
@@ -36,7 +37,6 @@ class Retry(object):
     Errors will be wrapped in :class:`~urllib3.exceptions.MaxRetryError` unless
     retries are disabled, in which case the causing exception will be raised.
 
-
     :param int total:
         Total number of retries to allow. Takes precedence over other counts.
 
@@ -184,8 +184,8 @@ class Retry(object):
         return isinstance(err, ConnectTimeoutError)
 
     def _is_read_error(self, err):
-        """ Errors that occur after the request has been started, so we can't
-        assume that the server did not process any of it.
+        """ Errors that occur after the request has been started, so we should
+        assume that the server began processing it.
         """
         return isinstance(err, (ReadTimeoutError, ProtocolError))
 
@@ -198,8 +198,7 @@ class Retry(object):
         return self.status_forcelist and status_code in self.status_forcelist
 
     def is_exhausted(self):
-        """ Are we out of retries?
-        """
+        """ Are we out of retries? """
         retry_counts = (self.total, self.connect, self.read, self.redirect)
         retry_counts = list(filter(None, retry_counts))
         if not retry_counts:
@@ -230,6 +229,7 @@ class Retry(object):
         connect = self.connect
         read = self.read
         redirect = self.redirect
+        cause = 'unknown'
 
         if error and self._is_connection_error(error):
             # Connect retry?
@@ -251,10 +251,16 @@ class Retry(object):
             # Redirect retry?
             if redirect is not None:
                 redirect -= 1
+            cause = 'too many redirects'
 
         else:
-            # FIXME: Nothing changed, scenario doesn't make sense.
+            # Incrementing because of a server error like a 500 in
+            # status_forcelist and a the given method is in the whitelist
             _observed_errors += 1
+            cause = ResponseError.GENERIC_ERROR
+            if response and response.status:
+                cause = ResponseError.SPECIFIC_ERROR.format(
+                    status_code=response.status)
 
         new_retry = self.new(
             total=total,
@@ -262,7 +268,7 @@ class Retry(object):
             _observed_errors=_observed_errors)
 
         if new_retry.is_exhausted():
-            raise MaxRetryError(_pool, url, error)
+            raise MaxRetryError(_pool, url, error or ResponseError(cause))
 
         log.debug("Incremented Retry for (url='%s'): %r" % (url, new_retry))
 

libs/requests/packages/urllib3/util/ssl_.py

@@ -4,18 +4,84 @@ from hashlib import md5, sha1
 from ..exceptions import SSLError
 
 
-try:  # Test for SSL features
-    SSLContext = None
-    HAS_SNI = False
+SSLContext = None
+HAS_SNI = False
+create_default_context = None
 
-    import ssl
+import errno
+import ssl
+
+try:  # Test for SSL features
     from ssl import wrap_socket, CERT_NONE, PROTOCOL_SSLv23
-    from ssl import SSLContext  # Modern SSL?
     from ssl import HAS_SNI  # Has SNI?
 except ImportError:
     pass
 
 
+try:
+    from ssl import OP_NO_SSLv2, OP_NO_SSLv3, OP_NO_COMPRESSION
+except ImportError:
+    OP_NO_SSLv2, OP_NO_SSLv3 = 0x1000000, 0x2000000
+    OP_NO_COMPRESSION = 0x20000
+
+try:
+    from ssl import _DEFAULT_CIPHERS
+except ImportError:
+    _DEFAULT_CIPHERS = (
+        'ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+HIGH:'
+        'DH+HIGH:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+HIGH:RSA+3DES:ECDH+RC4:'
+        'DH+RC4:RSA+RC4:!aNULL:!eNULL:!MD5'
+    )
+
+try:
+    from ssl import SSLContext  # Modern SSL?
+except ImportError:
+    import sys
+
+    class SSLContext(object):  # Platform-specific: Python 2 & 3.1
+        supports_set_ciphers = sys.version_info >= (2, 7)
+
+        def __init__(self, protocol_version):
+            self.protocol = protocol_version
+            # Use default values from a real SSLContext
+            self.check_hostname = False
+            self.verify_mode = ssl.CERT_NONE
+            self.ca_certs = None
+            self.options = 0
+            self.certfile = None
+            self.keyfile = None
+            self.ciphers = None
+
+        def load_cert_chain(self, certfile, keyfile):
+            self.certfile = certfile
+            self.keyfile = keyfile
+
+        def load_verify_locations(self, location):
+            self.ca_certs = location
+
+        def set_ciphers(self, cipher_suite):
+            if not self.supports_set_ciphers:
+                raise TypeError(
+                    'Your version of Python does not support setting '
+                    'a custom cipher suite. Please upgrade to Python '
+                    '2.7, 3.2, or later if you need this functionality.'
+                )
+            self.ciphers = cipher_suite
+
+        def wrap_socket(self, socket, server_hostname=None):
+            kwargs = {
+                'keyfile': self.keyfile,
+                'certfile': self.certfile,
+                'ca_certs': self.ca_certs,
+                'cert_reqs': self.verify_mode,
+                'ssl_version': self.protocol,
+            }
+            if self.supports_set_ciphers:  # Platform-specific: Python 2.7+
+                return wrap_socket(socket, ciphers=self.ciphers, **kwargs)
+            else:  # Platform-specific: Python 2.6
+                return wrap_socket(socket, **kwargs)
+
+
 def assert_fingerprint(cert, fingerprint):
     """
     Checks if given fingerprint matches the supplied certificate.
@@ -91,42 +157,98 @@ def resolve_ssl_version(candidate):
     return candidate
 
 
-if SSLContext is not None:  # Python 3.2+
-    def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None,
-                        ca_certs=None, server_hostname=None,
-                        ssl_version=None):
-        """
-        All arguments except `server_hostname` have the same meaning as for
-        :func:`ssl.wrap_socket`
+def create_urllib3_context(ssl_version=None, cert_reqs=ssl.CERT_REQUIRED,
+                           options=None, ciphers=None):
+    """All arguments have the same meaning as ``ssl_wrap_socket``.
 
-        :param server_hostname:
-            Hostname of the expected certificate
-        """
-        context = SSLContext(ssl_version)
-        context.verify_mode = cert_reqs
+    By default, this function does a lot of the same work that
+    ``ssl.create_default_context`` does on Python 3.4+. It:
 
-        # Disable TLS compression to migitate CRIME attack (issue #309)
-        OP_NO_COMPRESSION = 0x20000
-        context.options |= OP_NO_COMPRESSION
+    - Disables SSLv2, SSLv3, and compression
+    - Sets a restricted set of server ciphers
 
-        if ca_certs:
-            try:
-                context.load_verify_locations(ca_certs)
-            # Py32 raises IOError
-            # Py33 raises FileNotFoundError
-            except Exception as e:  # Reraise as SSLError
+    If you wish to enable SSLv3, you can do::
+
+        from urllib3.util import ssl_
+        context = ssl_.create_urllib3_context()
+        context.options &= ~ssl_.OP_NO_SSLv3
+
+    You can do the same to enable compression (substituting ``COMPRESSION``
+    for ``SSLv3`` in the last line above).
+
+    :param ssl_version:
+        The desired protocol version to use. This will default to
+        PROTOCOL_SSLv23 which will negotiate the highest protocol that both
+        the server and your installation of OpenSSL support.
+    :param cert_reqs:
+        Whether to require the certificate verification. This defaults to
+        ``ssl.CERT_REQUIRED``.
+    :param options:
+        Specific OpenSSL options. These default to ``ssl.OP_NO_SSLv2``,
+        ``ssl.OP_NO_SSLv3``, ``ssl.OP_NO_COMPRESSION``.
+    :param ciphers:
+        Which cipher suites to allow the server to select.
+    :returns:
+        Constructed SSLContext object with specified options
+    :rtype: SSLContext
+    """
+    context = SSLContext(ssl_version or ssl.PROTOCOL_SSLv23)
+
+    if options is None:
+        options = 0
+        # SSLv2 is easily broken and is considered harmful and dangerous
+        options |= OP_NO_SSLv2
+        # SSLv3 has several problems and is now dangerous
+        options |= OP_NO_SSLv3
+        # Disable compression to prevent CRIME attacks for OpenSSL 1.0+
+        # (issue #309)
+        options |= OP_NO_COMPRESSION
+
+    context.options |= options
+
+    if getattr(context, 'supports_set_ciphers', True):  # Platform-specific: Python 2.6
+        context.set_ciphers(ciphers or _DEFAULT_CIPHERS)
+
+    context.verify_mode = cert_reqs
+    if getattr(context, 'check_hostname', None) is not None:  # Platform-specific: Python 3.2
+        context.check_hostname = (context.verify_mode == ssl.CERT_REQUIRED)
+    return context
+
+
+def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None,
+                    ca_certs=None, server_hostname=None,
+                    ssl_version=None, ciphers=None, ssl_context=None):
+    """
+    All arguments except for server_hostname and ssl_context have the same
+    meaning as they do when using :func:`ssl.wrap_socket`.
+
+    :param server_hostname:
+        When SNI is supported, the expected hostname of the certificate
+    :param ssl_context:
+        A pre-made :class:`SSLContext` object. If none is provided, one will
+        be created using :func:`create_urllib3_context`.
+    :param ciphers:
+        A string of ciphers we wish the client to support. This is not
+        supported on Python 2.6 as the ssl module does not support it.
+    """
+    context = ssl_context
+    if context is None:
+        context = create_urllib3_context(ssl_version, cert_reqs,
+                                         ciphers=ciphers)
+
+    if ca_certs:
+        try:
+            context.load_verify_locations(ca_certs)
+        except IOError as e:  # Platform-specific: Python 2.6, 2.7, 3.2
+            raise SSLError(e)
+        # Py33 raises FileNotFoundError which subclasses OSError
+        # These are not equivalent unless we check the errno attribute
+        except OSError as e:  # Platform-specific: Python 3.3 and beyond
+            if e.errno == errno.ENOENT:
                 raise SSLError(e)
-        if certfile:
-            # FIXME: This block needs a test.
-            context.load_cert_chain(certfile, keyfile)
-        if HAS_SNI:  # Platform-specific: OpenSSL with enabled SNI
-            return context.wrap_socket(sock, server_hostname=server_hostname)
-        return context.wrap_socket(sock)
-
-else:  # Python 3.1 and earlier
-    def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None,
-                        ca_certs=None, server_hostname=None,
-                        ssl_version=None):
-        return wrap_socket(sock, keyfile=keyfile, certfile=certfile,
-                           ca_certs=ca_certs, cert_reqs=cert_reqs,
-                           ssl_version=ssl_version)
+            raise
+    if certfile:
+        context.load_cert_chain(certfile, keyfile)
+    if HAS_SNI:  # Platform-specific: OpenSSL with enabled SNI
+        return context.wrap_socket(sock, server_hostname=server_hostname)
+    return context.wrap_socket(sock)

libs/requests/packages/urllib3/util/url.py

@@ -40,6 +40,48 @@ class Url(namedtuple('Url', url_attrs)):
             return '%s:%d' % (self.host, self.port)
         return self.host
 
+    @property
+    def url(self):
+        """
+        Convert self into a url
+
+        This function should more or less round-trip with :func:`.parse_url`. The
+        returned url may not be exactly the same as the url inputted to
+        :func:`.parse_url`, but it should be equivalent by the RFC (e.g., urls
+        with a blank port will have : removed).
+
+        Example: ::
+
+            >>> U = parse_url('http://google.com/mail/')
+            >>> U.url
+            'http://google.com/mail/'
+            >>> Url('http', 'username:password', 'host.com', 80,
+            ... '/path', 'query', 'fragment').url
+            'http://username:password@host.com:80/path?query#fragment'
+        """
+        scheme, auth, host, port, path, query, fragment = self
+        url = ''
+
+        # We use "is not None" we want things to happen with empty strings (or 0 port)
+        if scheme is not None:
+            url += scheme + '://'
+        if auth is not None:
+            url += auth + '@'
+        if host is not None:
+            url += host
+        if port is not None:
+            url += ':' + str(port)
+        if path is not None:
+            url += path
+        if query is not None:
+            url += '?' + query
+        if fragment is not None:
+            url += '#' + fragment
+
+        return url
+
+    def __str__(self):
+        return self.url
 
 def split_first(s, delims):
     """
@@ -84,7 +126,7 @@ def parse_url(url):
     Example::
 
         >>> parse_url('http://google.com/mail/')
-        Url(scheme='http', host='google.com', port=None, path='/', ...)
+        Url(scheme='http', host='google.com', port=None, path='/mail/', ...)
         >>> parse_url('google.com:80')
         Url(scheme=None, host='google.com', port=80, path=None, ...)
         >>> parse_url('/foo?bar')
@@ -162,7 +204,6 @@ def parse_url(url):
 
     return Url(scheme, auth, host, port, path, query, fragment)
 
-
 def get_host(url):
     """
     Deprecated. Use :func:`.parse_url` instead.

libs/requests/sessions.py

@@ -13,7 +13,7 @@ from collections import Mapping
 from datetime import datetime
 
 from .auth import _basic_auth_str
-from .compat import cookielib, OrderedDict, urljoin, urlparse, builtin_str
+from .compat import cookielib, OrderedDict, urljoin, urlparse
 from .cookies import (
     cookiejar_from_dict, extract_cookies_to_jar, RequestsCookieJar, merge_cookies)
 from .models import Request, PreparedRequest, DEFAULT_REDIRECT_LIMIT
@@ -21,6 +21,7 @@ from .hooks import default_hooks, dispatch_hook
 from .utils import to_key_val_list, default_headers, to_native_string
 from .exceptions import (
     TooManyRedirects, InvalidSchema, ChunkedEncodingError, ContentDecodingError)
+from .packages.urllib3._collections import RecentlyUsedContainer
 from .structures import CaseInsensitiveDict
 
 from .adapters import HTTPAdapter
@@ -35,6 +36,8 @@ from .status_codes import codes
 # formerly defined here, reexposed here for backward compatibility
 from .models import REDIRECT_STATI
 
+REDIRECT_CACHE_SIZE = 1000
+
 
 def merge_setting(request_setting, session_setting, dict_class=OrderedDict):
     """
@@ -128,14 +131,14 @@ class SessionRedirectMixin(object):
             # Facilitate relative 'location' headers, as allowed by RFC 7231.
             # (e.g. '/path/to/resource' instead of 'http://domain.tld/path/to/resource')
             # Compliant with RFC3986, we percent encode the url.
-            if not urlparse(url).netloc:
+            if not parsed.netloc:
                 url = urljoin(resp.url, requote_uri(url))
             else:
                 url = requote_uri(url)
 
             prepared_request.url = to_native_string(url)
-            # cache the url
-            if resp.is_permanent_redirect:
+            # Cache the url, unless it redirects to itself.
+            if resp.is_permanent_redirect and req.url != prepared_request.url:
                 self.redirect_cache[req.url] = prepared_request.url
 
             # http://tools.ietf.org/html/rfc7231#section-6.4.4
@@ -271,9 +274,10 @@ class Session(SessionRedirectMixin):
     """
 
     __attrs__ = [
-        'headers', 'cookies', 'auth', 'timeout', 'proxies', 'hooks',
-        'params', 'verify', 'cert', 'prefetch', 'adapters', 'stream',
-        'trust_env', 'max_redirects', 'redirect_cache']
+        'headers', 'cookies', 'auth', 'proxies', 'hooks', 'params', 'verify',
+        'cert', 'prefetch', 'adapters', 'stream', 'trust_env',
+        'max_redirects',
+    ]
 
     def __init__(self):
 
@@ -326,7 +330,8 @@ class Session(SessionRedirectMixin):
         self.mount('https://', HTTPAdapter())
         self.mount('http://', HTTPAdapter())
 
-        self.redirect_cache = {}
+        # Only store 1000 redirects to prevent using infinite memory
+        self.redirect_cache = RecentlyUsedContainer(REDIRECT_CACHE_SIZE)
 
     def __enter__(self):
         return self
@@ -365,6 +370,7 @@ class Session(SessionRedirectMixin):
             url=request.url,
             files=request.files,
             data=request.data,
+            json=request.json,
             headers=merge_setting(request.headers, self.headers, dict_class=CaseInsensitiveDict),
             params=merge_setting(request.params, self.params),
             auth=merge_setting(auth, self.auth),
@@ -386,7 +392,8 @@ class Session(SessionRedirectMixin):
         hooks=None,
         stream=None,
         verify=None,
-        cert=None):
+        cert=None,
+        json=None):
         """Constructs a :class:`Request <Request>`, prepares it and sends it.
         Returns :class:`Response <Response>` object.
 
@@ -396,17 +403,22 @@ class Session(SessionRedirectMixin):
             string for the :class:`Request`.
         :param data: (optional) Dictionary or bytes to send in the body of the
             :class:`Request`.
+        :param json: (optional) json to send in the body of the
+            :class:`Request`.
         :param headers: (optional) Dictionary of HTTP Headers to send with the
             :class:`Request`.
         :param cookies: (optional) Dict or CookieJar object to send with the
             :class:`Request`.
-        :param files: (optional) Dictionary of 'filename': file-like-objects
+        :param files: (optional) Dictionary of ``'filename': file-like-objects``
             for multipart encoding upload.
         :param auth: (optional) Auth tuple or callable to enable
             Basic/Digest/Custom HTTP Auth.
-        :param timeout: (optional) Float describing the timeout of the
-            request in seconds.
-        :param allow_redirects: (optional) Boolean. Set to True by default.
+        :param timeout: (optional) How long to wait for the server to send
+            data before giving up, as a float, or a (`connect timeout, read
+            timeout <user/advanced.html#timeouts>`_) tuple.
+        :type timeout: float or tuple
+        :param allow_redirects: (optional) Set to True by default.
+        :type allow_redirects: bool
         :param proxies: (optional) Dictionary mapping protocol to the URL of
             the proxy.
         :param stream: (optional) whether to immediately download the response
@@ -417,7 +429,7 @@ class Session(SessionRedirectMixin):
             If Tuple, ('cert', 'key') pair.
         """
 
-        method = builtin_str(method)
+        method = to_native_string(method)
 
         # Create the Request.
         req = Request(
@@ -426,6 +438,7 @@ class Session(SessionRedirectMixin):
             headers = headers,
             files = files,
             data = data or {},
+            json = json,
             params = params or {},
             auth = auth,
             cookies = cookies,
@@ -479,15 +492,16 @@ class Session(SessionRedirectMixin):
         kwargs.setdefault('allow_redirects', False)
         return self.request('HEAD', url, **kwargs)
 
-    def post(self, url, data=None, **kwargs):
+    def post(self, url, data=None, json=None, **kwargs):
         """Sends a POST request. Returns :class:`Response` object.
 
         :param url: URL for the new :class:`Request` object.
         :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`.
+        :param json: (optional) json to send in the body of the :class:`Request`.
         :param \*\*kwargs: Optional arguments that ``request`` takes.
         """
 
-        return self.request('POST', url, data=data, **kwargs)
+        return self.request('POST', url, data=data, json=json, **kwargs)
 
     def put(self, url, data=None, **kwargs):
         """Sends a PUT request. Returns :class:`Response` object.
@@ -532,12 +546,13 @@ class Session(SessionRedirectMixin):
         if not isinstance(request, PreparedRequest):
             raise ValueError('You can only send PreparedRequests.')
 
-        redirect_count = 0
+        checked_urls = set()
         while request.url in self.redirect_cache:
-            redirect_count += 1
-            if redirect_count > self.max_redirects:
-                raise TooManyRedirects
-            request.url = self.redirect_cache.get(request.url)
+            checked_urls.add(request.url)
+            new_url = self.redirect_cache.get(request.url)
+            if new_url in checked_urls:
+                break
+            request.url = new_url
 
         # Set up variables needed for resolve_redirects and dispatching of hooks
         allow_redirects = kwargs.pop('allow_redirects', True)
@@ -647,12 +662,19 @@ class Session(SessionRedirectMixin):
             self.adapters[key] = self.adapters.pop(key)
 
     def __getstate__(self):
-        return dict((attr, getattr(self, attr, None)) for attr in self.__attrs__)
+        state = dict((attr, getattr(self, attr, None)) for attr in self.__attrs__)
+        state['redirect_cache'] = dict(self.redirect_cache)
+        return state
 
     def __setstate__(self, state):
+        redirect_cache = state.pop('redirect_cache', {})
         for attr, value in state.items():
             setattr(self, attr, value)
 
+        self.redirect_cache = RecentlyUsedContainer(REDIRECT_CACHE_SIZE)
+        for redirect, to in redirect_cache.items():
+            self.redirect_cache[redirect] = to
+
 
 def session():
     """Returns a :class:`Session` for context-management."""

libs/requests/utils.py

@@ -19,6 +19,7 @@ import re
 import sys
 import socket
 import struct
+import warnings
 
 from . import __version__
 from . import certs
@@ -114,7 +115,7 @@ def get_netrc_auth(url):
 def guess_filename(obj):
     """Tries to guess the filename of the given object."""
     name = getattr(obj, 'name', None)
-    if name and name[0] != '<' and name[-1] != '>':
+    if name and isinstance(name, builtin_str) and name[0] != '<' and name[-1] != '>':
         return os.path.basename(name)
 
 
@@ -287,6 +288,11 @@ def get_encodings_from_content(content):
 
     :param content: bytestring to extract encodings from.
     """
+    warnings.warn((
+        'In requests 3.0, get_encodings_from_content will be removed. For '
+        'more information, please see the discussion on issue #2266. (This'
+        ' warning should only appear once.)'),
+        DeprecationWarning)
 
     charset_re = re.compile(r'<meta.*?charset=["\']*(.+?)["\'>]', flags=re.I)
     pragma_re = re.compile(r'<meta.*?content=["\']*;?charset=(.+?)["\'>]', flags=re.I)
@@ -351,12 +357,14 @@ def get_unicode_from_response(r):
     Tried:
 
     1. charset from content-type
-
-    2. every encodings from ``<meta ... charset=XXX>``
-
-    3. fall back and replace all unicode characters
+    2. fall back and replace all unicode characters
 
     """
+    warnings.warn((
+        'In requests 3.0, get_unicode_from_response will be removed. For '
+        'more information, please see the discussion on issue #2266. (This'
+        ' warning should only appear once.)'),
+        DeprecationWarning)
 
     tried_encodings = []
 
@@ -555,7 +563,7 @@ def default_headers():
         'User-Agent': default_user_agent(),
         'Accept-Encoding': ', '.join(('gzip', 'deflate')),
         'Accept': '*/*',
-        'Connection': 'keep-alive'
+        'Connection': 'keep-alive',
     })
 
 
@@ -570,7 +578,7 @@ def parse_header_links(value):
 
     replace_chars = " '\""
 
-    for val in value.split(","):
+    for val in re.split(", *<", value):
         try:
             url, params = val.split(";", 1)
         except ValueError:
@@ -672,3 +680,18 @@ def to_native_string(string, encoding='ascii'):
             out = string.decode(encoding)
 
     return out
+
+
+def urldefragauth(url):
+    """
+    Given a url remove the fragment and the authentication part
+    """
+    scheme, netloc, path, params, query, fragment = urlparse(url)
+
+    # see func:`prepend_scheme_if_needed`
+    if not netloc:
+        netloc, path = path, netloc
+
+    netloc = netloc.rsplit('@', 1)[-1]
+
+    return urlunparse((scheme, netloc, path, params, query, ''))

libs/subliminal/services/podnapisi.py

@@ -1,110 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
-#
-# This file is part of subliminal.
-#
-# subliminal is free software; you can redistribute it and/or modify it under
-# the terms of the GNU Lesser General Public License as published by
-# the Free Software Foundation; either version 3 of the License, or
-# (at your option) any later version.
-#
-# subliminal is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with subliminal.  If not, see <http://www.gnu.org/licenses/>.
-from . import ServiceBase
-from ..exceptions import ServiceError, DownloadFailedError
-from ..language import language_set, Language
-from ..subtitles import get_subtitle_path, ResultSubtitle
-from ..utils import to_unicode
-from ..videos import Episode, Movie
-from hashlib import md5, sha256
-import logging
-import xmlrpclib
-
-
-logger = logging.getLogger(__name__)
-
-
-class Podnapisi(ServiceBase):
-    server_url = 'http://ssp.podnapisi.net:8000'
-    api_based = True
-    languages = language_set(['ar', 'be', 'bg', 'bs', 'ca', 'ca', 'cs', 'da', 'de', 'el', 'en',
-                              'es', 'et', 'fa', 'fi', 'fr', 'ga', 'he', 'hi', 'hr', 'hu', 'id',
-                              'is', 'it', 'ja', 'ko', 'lt', 'lv', 'mk', 'ms', 'nl', 'nn', 'pl',
-                              'pt', 'ro', 'ru', 'sk', 'sl', 'sq', 'sr', 'sv', 'th', 'tr', 'uk',
-                              'vi', 'zh', 'es-ar', 'pt-br'])
-    language_map = {'jp': Language('jpn'), Language('jpn'): 'jp',
-                    'gr': Language('gre'), Language('gre'): 'gr',
-                    'pb': Language('por-BR'), Language('por-BR'): 'pb',
-                    'ag': Language('spa-AR'), Language('spa-AR'): 'ag',
-                    'cyr': Language('srp')}
-    videos = [Episode, Movie]
-    require_video = True
-
-    def __init__(self, config=None):
-        super(Podnapisi, self).__init__(config)
-        self.server = xmlrpclib.ServerProxy(self.server_url)
-        self.token = None
-
-    def init(self):
-        super(Podnapisi, self).init()
-        result = self.server.initiate(self.user_agent)
-        if result['status'] != 200:
-            raise ServiceError('Initiate failed')
-        username = 'python_subliminal'
-        password = sha256(md5('XWFXQ6gE5Oe12rv4qxXX').hexdigest() + result['nonce']).hexdigest()
-        self.token = result['session']
-        result = self.server.authenticate(self.token, username, password)
-        if result['status'] != 200:
-            raise ServiceError('Authenticate failed')
-
-    def terminate(self):
-        super(Podnapisi, self).terminate()
-
-    def query(self, filepath, languages, moviehash):
-        results = self.server.search(self.token, [moviehash])
-        if results['status'] != 200:
-            logger.error('Search failed with error code %d' % results['status'])
-            return []
-        if not results['results'] or not results['results'][moviehash]['subtitles']:
-            logger.debug(u'Could not find subtitles for %r with token %s' % (moviehash, self.token))
-            return []
-        subtitles = []
-        for result in results['results'][moviehash]['subtitles']:
-            language = self.get_language(result['lang'])
-            if language not in languages:
-                continue
-            path = get_subtitle_path(filepath, language, self.config.multi)
-            subtitle = ResultSubtitle(path, language, self.__class__.__name__.lower(), result['id'],
-                                      release=to_unicode(result['release']), confidence=result['weight'])
-            subtitles.append(subtitle)
-        if not subtitles:
-            return []
-        # Convert weight to confidence
-        max_weight = float(max([s.confidence for s in subtitles]))
-        min_weight = float(min([s.confidence for s in subtitles]))
-        for subtitle in subtitles:
-            if max_weight == 0 and min_weight == 0:
-                subtitle.confidence = 1.0
-            else:
-                subtitle.confidence = (subtitle.confidence - min_weight) / (max_weight - min_weight)
-        return subtitles
-
-    def list_checked(self, video, languages):
-        results = self.query(video.path, languages, video.hashes['OpenSubtitles'])
-        return results
-
-    def download(self, subtitle):
-        results = self.server.download(self.token, [subtitle.link])
-        if results['status'] != 200:
-            raise DownloadFailedError()
-        subtitle.link = 'http://www.podnapisi.net/static/podnapisi/' + results['names'][0]['filename']
-        self.download_file(subtitle.link, subtitle.path)
-        return subtitle
-
-
-Service = Podnapisi

libs/tornado/httpclient.py

@@ -95,7 +95,8 @@ class HTTPClient(object):
         If it is a string, we construct an `HTTPRequest` using any additional
         kwargs: ``HTTPRequest(request, **kwargs)``
 
-        If an error occurs during the fetch, we raise an `HTTPError`.
+        If an error occurs during the fetch, we raise an `HTTPError` unless
+        the ``raise_error`` keyword argument is set to False.
         """
         response = self._io_loop.run_sync(functools.partial(
             self._async_client.fetch, request, **kwargs))
@@ -200,7 +201,7 @@ class AsyncHTTPClient(Configurable):
                 raise RuntimeError("inconsistent AsyncHTTPClient cache")
             del self._instance_cache[self.io_loop]
 
-    def fetch(self, request, callback=None, **kwargs):
+    def fetch(self, request, callback=None, raise_error=True, **kwargs):
         """Executes a request, asynchronously returning an `HTTPResponse`.
 
         The request may be either a string URL or an `HTTPRequest` object.
@@ -208,8 +209,10 @@ class AsyncHTTPClient(Configurable):
         kwargs: ``HTTPRequest(request, **kwargs)``
 
         This method returns a `.Future` whose result is an
-        `HTTPResponse`.  The ``Future`` will raise an `HTTPError` if
-        the request returned a non-200 response code.
+        `HTTPResponse`.  By default, the ``Future`` will raise an `HTTPError`
+        if the request returned a non-200 response code. Instead, if
+        ``raise_error`` is set to False, the response will always be
+        returned regardless of the response code.
 
         If a ``callback`` is given, it will be invoked with the `HTTPResponse`.
         In the callback interface, `HTTPError` is not automatically raised.
@@ -243,7 +246,7 @@ class AsyncHTTPClient(Configurable):
             future.add_done_callback(handle_future)
 
         def handle_response(response):
-            if response.error:
+            if raise_error and response.error:
                 future.set_exception(response.error)
             else:
                 future.set_result(response)

libs/tornado/httpserver.py

@@ -42,30 +42,10 @@ from tornado.tcpserver import TCPServer
 class HTTPServer(TCPServer, httputil.HTTPServerConnectionDelegate):
     r"""A non-blocking, single-threaded HTTP server.
 
-    A server is defined by either a request callback that takes a
-    `.HTTPServerRequest` as an argument or a `.HTTPServerConnectionDelegate`
-    instance.
-
-    A simple example server that echoes back the URI you requested::
-
-        import tornado.httpserver
-        import tornado.ioloop
-        from tornado import httputil
-
-        def handle_request(request):
-           message = "You requested %s\n" % request.uri
-           request.connection.write_headers(
-               httputil.ResponseStartLine('HTTP/1.1', 200, 'OK'),
-               httputil.HTTPHeaders({"Content-Length": str(len(message))}))
-           request.connection.write(message)
-           request.connection.finish()
-
-        http_server = tornado.httpserver.HTTPServer(handle_request)
-        http_server.listen(8888)
-        tornado.ioloop.IOLoop.instance().start()
-
-    Applications should use the methods of `.HTTPConnection` to write
-    their response.
+    A server is defined by a subclass of `.HTTPServerConnectionDelegate`,
+    or, for backwards compatibility, a callback that takes an
+    `.HTTPServerRequest` as an argument. The delegate is usually a
+    `tornado.web.Application`.
 
     `HTTPServer` supports keep-alive connections by default
     (automatically for HTTP/1.1, or for HTTP/1.0 when the client

libs/tornado/httputil.py

@@ -331,7 +331,7 @@ class HTTPServerRequest(object):
         self.uri = uri
         self.version = version
         self.headers = headers or HTTPHeaders()
-        self.body = body or ""
+        self.body = body or b""
 
         # set remote IP and protocol
         context = getattr(connection, 'context', None)
@@ -873,3 +873,17 @@ def _encode_header(key, pdict):
 def doctests():
     import doctest
     return doctest.DocTestSuite()
+
+def split_host_and_port(netloc):
+    """Returns ``(host, port)`` tuple from ``netloc``.
+
+    Returned ``port`` will be ``None`` if not present.
+    """
+    match = re.match(r'^(.+):(\d+)$', netloc)
+    if match:
+        host = match.group(1)
+        port = int(match.group(2))
+    else:
+        host = netloc
+        port = None
+    return (host, port)

libs/tornado/iostream.py

@@ -331,7 +331,7 @@ class BaseIOStream(object):
         if data:
             if (self.max_write_buffer_size is not None and
                     self._write_buffer_size + len(data) > self.max_write_buffer_size):
-                raise StreamBufferFullError("Reached maximum read buffer size")
+                raise StreamBufferFullError("Reached maximum write buffer size")
             # Break up large contiguous strings before inserting them in the
             # write buffer, so we don't have to recopy the entire thing
             # as we slice off pieces to send to the socket.

libs/tornado/netutil.py

@@ -20,7 +20,7 @@ from __future__ import absolute_import, division, print_function, with_statement
 
 import errno
 import os
-import platform
+import sys
 import socket
 import stat
 
@@ -105,7 +105,7 @@ def bind_sockets(port, address=None, family=socket.AF_UNSPEC,
     for res in set(socket.getaddrinfo(address, port, family, socket.SOCK_STREAM,
                                       0, flags)):
         af, socktype, proto, canonname, sockaddr = res
-        if (platform.system() == 'Darwin' and address == 'localhost' and
+        if (sys.platform == 'darwin' and address == 'localhost' and
                 af == socket.AF_INET6 and sockaddr[3] != 0):
             # Mac OS X includes a link-local address fe80::1%lo0 in the
             # getaddrinfo results for 'localhost'.  However, the firewall

libs/tornado/options.py

@@ -204,6 +204,13 @@ class OptionParser(object):
                         (name, self._options[name].file_name))
         frame = sys._getframe(0)
         options_file = frame.f_code.co_filename
+
+        # Can be called directly, or through top level define() fn, in which
+        # case, step up above that frame to look for real caller.
+        if (frame.f_back.f_code.co_filename == options_file and
+                frame.f_back.f_code.co_name == 'define'):
+            frame = frame.f_back
+
         file_name = frame.f_back.f_code.co_filename
         if file_name == options_file:
             file_name = ""

libs/tornado/platform/kqueue.py

@@ -54,8 +54,7 @@ class _KQueue(object):
         if events & IOLoop.WRITE:
             kevents.append(select.kevent(
                 fd, filter=select.KQ_FILTER_WRITE, flags=flags))
-        if events & IOLoop.READ or not kevents:
-            # Always read when there is not a write
+        if events & IOLoop.READ:
             kevents.append(select.kevent(
                 fd, filter=select.KQ_FILTER_READ, flags=flags))
         # Even though control() takes a list, it seems to return EINVAL

libs/tornado/platform/select.py

@@ -47,7 +47,7 @@ class _Select(object):
             # Closed connections are reported as errors by epoll and kqueue,
             # but as zero-byte reads by select, so when errors are requested
             # we need to listen for both read and error.
-            self.read_fds.add(fd)
+            #self.read_fds.add(fd)
 
     def modify(self, fd, events):
         self.unregister(fd)

libs/tornado/simple_httpclient.py

@@ -193,12 +193,8 @@ class _HTTPConnection(httputil.HTTPMessageDelegate):
             netloc = self.parsed.netloc
             if "@" in netloc:
                 userpass, _, netloc = netloc.rpartition("@")
-            match = re.match(r'^(.+):(\d+)$', netloc)
-            if match:
-                host = match.group(1)
-                port = int(match.group(2))
-            else:
-                host = netloc
+            host, port = httputil.split_host_and_port(netloc)
+            if port is None:
                 port = 443 if self.parsed.scheme == "https" else 80
             if re.match(r'^\[.*\]$', host):
                 # raw ipv6 addresses in urls are enclosed in brackets

libs/tornado/testing.py

@@ -19,6 +19,7 @@ try:
     from tornado.simple_httpclient import SimpleAsyncHTTPClient
     from tornado.ioloop import IOLoop, TimeoutError
     from tornado import netutil
+    from tornado.process import Subprocess
 except ImportError:
     # These modules are not importable on app engine.  Parts of this module
     # won't work, but e.g. LogTrapTestCase and main() will.
@@ -28,6 +29,7 @@ except ImportError:
     IOLoop = None
     netutil = None
     SimpleAsyncHTTPClient = None
+    Subprocess = None
 from tornado.log import gen_log, app_log
 from tornado.stack_context import ExceptionStackContext
 from tornado.util import raise_exc_info, basestring_type
@@ -214,6 +216,8 @@ class AsyncTestCase(unittest.TestCase):
         self.io_loop.make_current()
 
     def tearDown(self):
+        # Clean up Subprocess, so it can be used again with a new ioloop.
+        Subprocess.uninitialize()
         self.io_loop.clear_current()
         if (not IOLoop.initialized() or
                 self.io_loop is not IOLoop.instance()):

libs/tornado/web.py

@@ -85,6 +85,7 @@ from tornado import stack_context
 from tornado import template
 from tornado.escape import utf8, _unicode
 from tornado.util import import_object, ObjectDict, raise_exc_info, unicode_type, _websocket_mask
+from tornado.httputil import split_host_and_port
 
 
 try:
@@ -1477,7 +1478,7 @@ def asynchronous(method):
         with stack_context.ExceptionStackContext(
                 self._stack_context_handle_exception):
             result = method(self, *args, **kwargs)
-            if isinstance(result, Future):
+            if is_future(result):
                 # If @asynchronous is used with @gen.coroutine, (but
                 # not @gen.engine), we can automatically finish the
                 # request when the future resolves.  Additionally,
@@ -1518,7 +1519,7 @@ def stream_request_body(cls):
       the entire body has been read.
 
     There is a subtle interaction between ``data_received`` and asynchronous
-    ``prepare``: The first call to ``data_recieved`` may occur at any point
+    ``prepare``: The first call to ``data_received`` may occur at any point
     after the call to ``prepare`` has returned *or yielded*.
     """
     if not issubclass(cls, RequestHandler):
@@ -1729,7 +1730,7 @@ class Application(httputil.HTTPServerConnectionDelegate):
         self.transforms.append(transform_class)
 
     def _get_host_handlers(self, request):
-        host = request.host.lower().split(':')[0]
+        host = split_host_and_port(request.host.lower())[0]
         matches = []
         for pattern, handlers in self.handlers:
             if pattern.match(host):
@@ -1845,7 +1846,7 @@ class _RequestDispatcher(httputil.HTTPMessageDelegate):
         handlers = app._get_host_handlers(self.request)
         if not handlers:
             self.handler_class = RedirectHandler
-            self.handler_kwargs = dict(url="http://" + app.default_host + "/")
+            self.handler_kwargs = dict(url="%s://%s/" % (self.request.protocol, app.default_host))
             return
         for spec in handlers:
             match = spec.regex.match(self.request.path)

libs/tornado/websocket.py

@@ -229,7 +229,7 @@ class WebSocketHandler(tornado.web.RequestHandler):
         """
         return None
 
-    def open(self):
+    def open(self, *args, **kwargs):
         """Invoked when a new WebSocket is opened.
 
         The arguments to `open` are extracted from the `tornado.web.URLSpec`

libs/tornado/wsgi.py

@@ -207,7 +207,7 @@ class WSGIAdapter(object):
             body = environ["wsgi.input"].read(
                 int(headers["Content-Length"]))
         else:
-            body = ""
+            body = b""
         protocol = environ["wsgi.url_scheme"]
         remote_ip = environ.get("REMOTE_ADDR", "")
         if environ.get("HTTP_HOST"):