Merge branch 'develop'

couchpotato/core/media/_base/providers/nzb/newznab.py

@@ -20,7 +20,6 @@ log = CPLog(__name__)
 class Base(NZBProvider, RSS):
 
     urls = {
-        'detail': 'details/%s',
         'download': 't=get&id=%s'
     }
 
@@ -68,7 +67,8 @@ class Base(NZBProvider, RSS):
             if not date:
                 date = self.getTextElement(nzb, 'pubDate')
 
-            nzb_id = self.getTextElement(nzb, 'guid').split('/')[-1:].pop()
+            detail_url = self.getTextElement(nzb, 'guid')
+            nzb_id = detail_url.split('/')[-1:].pop()
             name = self.getTextElement(nzb, 'title')
 
             if not name:
@@ -103,7 +103,7 @@ class Base(NZBProvider, RSS):
                 'age': self.calculateAge(int(time.mktime(parse(date).timetuple()))),
                 'size': int(self.getElement(nzb, 'enclosure').attrib['length']) / 1024 / 1024,
                 'url': ((self.getUrl(host['host']) + self.urls['download']) % tryUrlencode(nzb_id)) + self.getApiExt(host),
-                'detail_url': (cleanHost(host['host']) + self.urls['detail']) % tryUrlencode(nzb_id),
+                'detail_url': detail_url,
                 'content': self.getTextElement(nzb, 'description'),
                 'description': description,
                 'score': host['extra_score'],

libs/chardet/__init__.py

@@ -15,7 +15,7 @@
 # 02110-1301  USA
 ######################### END LICENSE BLOCK #########################
 
-__version__ = "2.2.1"
+__version__ = "2.3.0"
 from sys import version_info
 
 

libs/chardet/chardetect.py

@@ -12,34 +12,68 @@ Example::
 If no paths are provided, it takes its input from stdin.
 
 """
-from io import open
-from sys import argv, stdin
 
+from __future__ import absolute_import, print_function, unicode_literals
+
+import argparse
+import sys
+from io import open
+
+from chardet import __version__
 from chardet.universaldetector import UniversalDetector
 
 
-def description_of(file, name='stdin'):
+def description_of(lines, name='stdin'):
-    """Return a string describing the probable encoding of a file."""
+    """
+    Return a string describing the probable encoding of a file or
+    list of strings.
+
+    :param lines: The lines to get the encoding of.
+    :type lines: Iterable of bytes
+    :param name: Name of file or collection of lines
+    :type name: str
+    """
     u = UniversalDetector()
-    for line in file:
+    for line in lines:
         u.feed(line)
     u.close()
     result = u.result
     if result['encoding']:
-        return '%s: %s with confidence %s' % (name,
+        return '{0}: {1} with confidence {2}'.format(name, result['encoding'],
-                                              result['encoding'],
+                                                     result['confidence'])
-                                              result['confidence'])
     else:
-        return '%s: no result' % name
+        return '{0}: no result'.format(name)
 
 
-def main():
+def main(argv=None):
-    if len(argv) <= 1:
+    '''
-        print(description_of(stdin))
+    Handles command line arguments and gets things started.
-    else:
+
-        for path in argv[1:]:
+    :param argv: List of arguments, as if specified on the command-line.
-            with open(path, 'rb') as f:
+                 If None, ``sys.argv[1:]`` is used instead.
-                print(description_of(f, path))
+    :type argv: list of str
+    '''
+    # Get command line arguments
+    parser = argparse.ArgumentParser(
+        description="Takes one or more file paths and reports their detected \
+                     encodings",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        conflict_handler='resolve')
+    parser.add_argument('input',
+                        help='File whose encoding we would like to determine.',
+                        type=argparse.FileType('rb'), nargs='*',
+                        default=[sys.stdin])
+    parser.add_argument('--version', action='version',
+                        version='%(prog)s {0}'.format(__version__))
+    args = parser.parse_args(argv)
+
+    for f in args.input:
+        if f.isatty():
+            print("You are running chardetect interactively. Press " +
+                  "CTRL-D twice at the start of a blank line to signal the " +
+                  "end of your input. If you want help, run chardetect " +
+                  "--help\n", file=sys.stderr)
+        print(description_of(f, f.name))
 
 
 if __name__ == '__main__':

libs/chardet/jpcntx.py

@@ -177,6 +177,12 @@ class JapaneseContextAnalysis:
         return -1, 1
 
 class SJISContextAnalysis(JapaneseContextAnalysis):
+    def __init__(self):
+        self.charset_name = "SHIFT_JIS"
+
+    def get_charset_name(self):
+        return self.charset_name
+
     def get_order(self, aBuf):
         if not aBuf:
             return -1, 1
@@ -184,6 +190,8 @@ class SJISContextAnalysis(JapaneseContextAnalysis):
         first_char = wrap_ord(aBuf[0])
         if ((0x81 <= first_char <= 0x9F) or (0xE0 <= first_char <= 0xFC)):
             charLen = 2
+            if (first_char == 0x87) or (0xFA <= first_char <= 0xFC):
+                self.charset_name = "CP932"
         else:
             charLen = 1
 

libs/chardet/latin1prober.py

@@ -129,11 +129,11 @@ class Latin1Prober(CharSetProber):
         if total < 0.01:
             confidence = 0.0
         else:
-            confidence = ((self._mFreqCounter[3] / total)
+            confidence = ((self._mFreqCounter[3] - self._mFreqCounter[1] * 20.0)
-                          - (self._mFreqCounter[1] * 20.0 / total))
+                          / total)
         if confidence < 0.0:
             confidence = 0.0
         # lower the confidence of latin1 so that other more accurate
         # detector can take priority.
-        confidence = confidence * 0.5
+        confidence = confidence * 0.73
         return confidence

libs/chardet/mbcssm.py

@@ -353,7 +353,7 @@ SJIS_cls = (
     2,2,2,2,2,2,2,2,  # 68 - 6f
     2,2,2,2,2,2,2,2,  # 70 - 77
     2,2,2,2,2,2,2,1,  # 78 - 7f
-    3,3,3,3,3,3,3,3,  # 80 - 87
+    3,3,3,3,3,2,2,3,  # 80 - 87
     3,3,3,3,3,3,3,3,  # 88 - 8f
     3,3,3,3,3,3,3,3,  # 90 - 97
     3,3,3,3,3,3,3,3,  # 98 - 9f
@@ -369,9 +369,8 @@ SJIS_cls = (
     2,2,2,2,2,2,2,2,  # d8 - df
     3,3,3,3,3,3,3,3,  # e0 - e7
     3,3,3,3,3,4,4,4,  # e8 - ef
-    4,4,4,4,4,4,4,4,  # f0 - f7
+    3,3,3,3,3,3,3,3,  # f0 - f7
-    4,4,4,4,4,0,0,0   # f8 - ff
+    3,3,3,3,3,0,0,0)  # f8 - ff
-)
 
 
 SJIS_st = (
@@ -571,5 +570,3 @@ UTF8SMModel = {'classTable': UTF8_cls,
                'stateTable': UTF8_st,
                'charLenTable': UTF8CharLenTable,
                'name': 'UTF-8'}
-
-# flake8: noqa

libs/chardet/sjisprober.py

@@ -47,7 +47,7 @@ class SJISProber(MultiByteCharSetProber):
         self._mContextAnalyzer.reset()
 
     def get_charset_name(self):
-        return "SHIFT_JIS"
+        return self._mContextAnalyzer.get_charset_name()
 
     def feed(self, aBuf):
         aLen = len(aBuf)

libs/chardet/universaldetector.py

@@ -71,9 +71,9 @@ class UniversalDetector:
 
         if not self._mGotData:
             # If the data starts with BOM, we know it is UTF
-            if aBuf[:3] == codecs.BOM:
+            if aBuf[:3] == codecs.BOM_UTF8:
                 # EF BB BF  UTF-8 with BOM
-                self.result = {'encoding': "UTF-8", 'confidence': 1.0}
+                self.result = {'encoding': "UTF-8-SIG", 'confidence': 1.0}
             elif aBuf[:4] == codecs.BOM_UTF32_LE:
                 # FF FE 00 00  UTF-32, little-endian BOM
                 self.result = {'encoding': "UTF-32LE", 'confidence': 1.0}

libs/requests/__init__.py

@@ -13,7 +13,7 @@ Requests is an HTTP library, written in Python, for human beings. Basic GET
 usage:
 
    >>> import requests
-   >>> r = requests.get('http://python.org')
+   >>> r = requests.get('https://www.python.org')
    >>> r.status_code
    200
    >>> 'Python is a programming language' in r.content
@@ -22,7 +22,7 @@ usage:
 ... or POST:
 
    >>> payload = dict(key1='value1', key2='value2')
-   >>> r = requests.post("http://httpbin.org/post", data=payload)
+   >>> r = requests.post('http://httpbin.org/post', data=payload)
    >>> print(r.text)
    {
      ...
@@ -42,8 +42,8 @@ is at <http://python-requests.org>.
 """
 
 __title__ = 'requests'
-__version__ = '2.4.0'
+__version__ = '2.5.1'
-__build__ = 0x020400
+__build__ = 0x020501
 __author__ = 'Kenneth Reitz'
 __license__ = 'Apache 2.0'
 __copyright__ = 'Copyright 2014 Kenneth Reitz'

libs/requests/adapters.py

@@ -15,19 +15,21 @@ from .packages.urllib3 import Retry
 from .packages.urllib3.poolmanager import PoolManager, proxy_from_url
 from .packages.urllib3.response import HTTPResponse
 from .packages.urllib3.util import Timeout as TimeoutSauce
-from .compat import urlparse, basestring, urldefrag
+from .compat import urlparse, basestring
 from .utils import (DEFAULT_CA_BUNDLE_PATH, get_encoding_from_headers,
-                    prepend_scheme_if_needed, get_auth_from_url)
+                    prepend_scheme_if_needed, get_auth_from_url, urldefragauth)
 from .structures import CaseInsensitiveDict
 from .packages.urllib3.exceptions import ConnectTimeoutError
 from .packages.urllib3.exceptions import HTTPError as _HTTPError
 from .packages.urllib3.exceptions import MaxRetryError
 from .packages.urllib3.exceptions import ProxyError as _ProxyError
+from .packages.urllib3.exceptions import ProtocolError
 from .packages.urllib3.exceptions import ReadTimeoutError
 from .packages.urllib3.exceptions import SSLError as _SSLError
+from .packages.urllib3.exceptions import ResponseError
 from .cookies import extract_cookies_to_jar
 from .exceptions import (ConnectionError, ConnectTimeout, ReadTimeout, SSLError,
-                         ProxyError)
+                         ProxyError, RetryError)
 from .auth import _basic_auth_str
 
 DEFAULT_POOLBLOCK = False
@@ -59,8 +61,12 @@ class HTTPAdapter(BaseAdapter):
     :param pool_connections: The number of urllib3 connection pools to cache.
     :param pool_maxsize: The maximum number of connections to save in the pool.
     :param int max_retries: The maximum number of retries each connection
-        should attempt. Note, this applies only to failed connections and
+        should attempt. Note, this applies only to failed DNS lookups, socket
-        timeouts, never to requests where the server returns a response.
+        connections and connection timeouts, never to requests where data has
+        made it to the server. By default, Requests does not retry failed
+        connections. If you need granular control over the conditions under
+        which we retry a request, import urllib3's ``Retry`` class and pass
+        that instead.
     :param pool_block: Whether the connection pool should block for connections.
 
     Usage::
@@ -76,7 +82,10 @@ class HTTPAdapter(BaseAdapter):
     def __init__(self, pool_connections=DEFAULT_POOLSIZE,
                  pool_maxsize=DEFAULT_POOLSIZE, max_retries=DEFAULT_RETRIES,
                  pool_block=DEFAULT_POOLBLOCK):
-        self.max_retries = max_retries
+        if max_retries == DEFAULT_RETRIES:
+            self.max_retries = Retry(0, read=False)
+        else:
+            self.max_retries = Retry.from_int(max_retries)
         self.config = {}
         self.proxy_manager = {}
 
@@ -122,7 +131,7 @@ class HTTPAdapter(BaseAdapter):
         self._pool_block = block
 
         self.poolmanager = PoolManager(num_pools=connections, maxsize=maxsize,
-                                       block=block, **pool_kwargs)
+                                       block=block, strict=True, **pool_kwargs)
 
     def proxy_manager_for(self, proxy, **proxy_kwargs):
         """Return urllib3 ProxyManager for the given proxy.
@@ -269,7 +278,7 @@ class HTTPAdapter(BaseAdapter):
         proxy = proxies.get(scheme)
 
         if proxy and scheme != 'https':
-            url, _ = urldefrag(request.url)
+            url = urldefragauth(request.url)
         else:
             url = request.path_url
 
@@ -316,8 +325,10 @@ class HTTPAdapter(BaseAdapter):
 
         :param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
         :param stream: (optional) Whether to stream the request content.
-        :param timeout: (optional) The timeout on the request.
+        :param timeout: (optional) How long to wait for the server to send
-        :type timeout: float or tuple (connect timeout, read timeout), eg (3.1, 20)
+            data before giving up, as a float, or a (`connect timeout, read
+            timeout <user/advanced.html#timeouts>`_) tuple.
+        :type timeout: float or tuple
         :param verify: (optional) Whether to verify SSL certificates.
         :param cert: (optional) Any user-provided SSL certificate to be trusted.
         :param proxies: (optional) The proxies dictionary to apply to the request.
@@ -355,7 +366,7 @@ class HTTPAdapter(BaseAdapter):
                     assert_same_host=False,
                     preload_content=False,
                     decode_content=False,
-                    retries=Retry(self.max_retries, read=False),
+                    retries=self.max_retries,
                     timeout=timeout
                 )
 
@@ -400,13 +411,16 @@ class HTTPAdapter(BaseAdapter):
                     # All is well, return the connection to the pool.
                     conn._put_conn(low_conn)
 
-        except socket.error as sockerr:
+        except (ProtocolError, socket.error) as err:
-            raise ConnectionError(sockerr, request=request)
+            raise ConnectionError(err, request=request)
 
         except MaxRetryError as e:
             if isinstance(e.reason, ConnectTimeoutError):
                 raise ConnectTimeout(e, request=request)
 
+            if isinstance(e.reason, ResponseError):
+                raise RetryError(e, request=request)
+
             raise ConnectionError(e, request=request)
 
         except _ProxyError as e:

libs/requests/api.py

@@ -22,12 +22,17 @@ def request(method, url, **kwargs):
     :param url: URL for the new :class:`Request` object.
     :param params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`.
     :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`.
+    :param json: (optional) json data to send in the body of the :class:`Request`.
     :param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`.
     :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`.
-    :param files: (optional) Dictionary of 'name': file-like-objects (or {'name': ('filename', fileobj)}) for multipart encoding upload.
+    :param files: (optional) Dictionary of ``'name': file-like-objects`` (or ``{'name': ('filename', fileobj)}``) for multipart encoding upload.
     :param auth: (optional) Auth tuple to enable Basic/Digest/Custom HTTP Auth.
-    :param timeout: (optional) Float describing the timeout of the request in seconds.
+    :param timeout: (optional) How long to wait for the server to send data
+        before giving up, as a float, or a (`connect timeout, read timeout
+        <user/advanced.html#timeouts>`_) tuple.
+    :type timeout: float or tuple
     :param allow_redirects: (optional) Boolean. Set to True if POST/PUT/DELETE redirect following is allowed.
+    :type allow_redirects: bool
     :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy.
     :param verify: (optional) if ``True``, the SSL cert will be verified. A CA_BUNDLE path can also be provided.
     :param stream: (optional) if ``False``, the response content will be immediately downloaded.
@@ -41,7 +46,12 @@ def request(method, url, **kwargs):
     """
 
     session = sessions.Session()
-    return session.request(method=method, url=url, **kwargs)
+    response = session.request(method=method, url=url, **kwargs)
+    # By explicitly closing the session, we avoid leaving sockets open which
+    # can trigger a ResourceWarning in some cases, and look like a memory leak
+    # in others.
+    session.close()
+    return response
 
 
 def get(url, **kwargs):
@@ -77,15 +87,16 @@ def head(url, **kwargs):
     return request('head', url, **kwargs)
 
 
-def post(url, data=None, **kwargs):
+def post(url, data=None, json=None, **kwargs):
     """Sends a POST request. Returns :class:`Response` object.
 
     :param url: URL for the new :class:`Request` object.
     :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`.
+    :param json: (optional) json data to send in the body of the :class:`Request`.
     :param \*\*kwargs: Optional arguments that ``request`` takes.
     """
 
-    return request('post', url, data=data, **kwargs)
+    return request('post', url, data=data, json=json, **kwargs)
 
 
 def put(url, data=None, **kwargs):

libs/requests/auth.py

@@ -17,6 +17,7 @@ from base64 import b64encode
 from .compat import urlparse, str
 from .cookies import extract_cookies_to_jar
 from .utils import parse_dict_header, to_native_string
+from .status_codes import codes
 
 CONTENT_TYPE_FORM_URLENCODED = 'application/x-www-form-urlencoded'
 CONTENT_TYPE_MULTI_PART = 'multipart/form-data'
@@ -66,6 +67,7 @@ class HTTPDigestAuth(AuthBase):
         self.nonce_count = 0
         self.chal = {}
         self.pos = None
+        self.num_401_calls = 1
 
     def build_digest_header(self, method, url):
 
@@ -150,6 +152,11 @@ class HTTPDigestAuth(AuthBase):
 
         return 'Digest %s' % (base)
 
+    def handle_redirect(self, r, **kwargs):
+        """Reset num_401_calls counter on redirects."""
+        if r.is_redirect:
+            self.num_401_calls = 1
+
     def handle_401(self, r, **kwargs):
         """Takes the given response and tries digest-auth, if needed."""
 
@@ -162,7 +169,7 @@ class HTTPDigestAuth(AuthBase):
 
         if 'digest' in s_auth.lower() and num_401_calls < 2:
 
-            setattr(self, 'num_401_calls', num_401_calls + 1)
+            self.num_401_calls += 1
             pat = re.compile(r'digest ', flags=re.IGNORECASE)
             self.chal = parse_dict_header(pat.sub('', s_auth, count=1))
 
@@ -182,7 +189,7 @@ class HTTPDigestAuth(AuthBase):
 
             return _r
 
-        setattr(self, 'num_401_calls', 1)
+        self.num_401_calls = 1
         return r
 
     def __call__(self, r):
@@ -192,6 +199,11 @@ class HTTPDigestAuth(AuthBase):
         try:
             self.pos = r.body.tell()
         except AttributeError:
-            pass
+            # In the case of HTTPDigestAuth being reused and the body of
+            # the previous request was a file-like object, pos has the
+            # file position of the previous body. Ensure it's set to
+            # None.
+            self.pos = None
         r.register_hook('response', self.handle_401)
+        r.register_hook('response', self.handle_redirect)
         return r

libs/requests/compat.py

@@ -76,7 +76,7 @@ is_solaris = ('solar==' in str(sys.platform).lower())   # Complete guess.
 try:
     import simplejson as json
 except (ImportError, SyntaxError):
-    # simplejson does not support Python 3.2, it thows a SyntaxError
+    # simplejson does not support Python 3.2, it throws a SyntaxError
     # because of u'...' Unicode literals.
     import json
 

libs/requests/exceptions.py

@@ -46,15 +46,16 @@ class SSLError(ConnectionError):
 class Timeout(RequestException):
     """The request timed out.
 
-    Catching this error will catch both :exc:`ConnectTimeout` and
+    Catching this error will catch both
-    :exc:`ReadTimeout` errors.
+    :exc:`~requests.exceptions.ConnectTimeout` and
+    :exc:`~requests.exceptions.ReadTimeout` errors.
     """
 
 
 class ConnectTimeout(ConnectionError, Timeout):
-    """The request timed out while trying to connect to the server.
+    """The request timed out while trying to connect to the remote server.
 
-    Requests that produce this error are safe to retry
+    Requests that produced this error are safe to retry.
     """
 
 
@@ -88,3 +89,11 @@ class ChunkedEncodingError(RequestException):
 
 class ContentDecodingError(RequestException, BaseHTTPError):
     """Failed to decode response content"""
+
+
+class StreamConsumedError(RequestException, TypeError):
+    """The content for this response was already consumed"""
+
+
+class RetryError(RequestException):
+    """Custom retries logic failed"""

libs/requests/models.py

@@ -20,10 +20,10 @@ from .packages.urllib3.fields import RequestField
 from .packages.urllib3.filepost import encode_multipart_formdata
 from .packages.urllib3.util import parse_url
 from .packages.urllib3.exceptions import (
-    DecodeError, ReadTimeoutError, ProtocolError)
+    DecodeError, ReadTimeoutError, ProtocolError, LocationParseError)
 from .exceptions import (
-    HTTPError, RequestException, MissingSchema, InvalidURL,
+    HTTPError, MissingSchema, InvalidURL, ChunkedEncodingError,
-    ChunkedEncodingError, ContentDecodingError, ConnectionError)
+    ContentDecodingError, ConnectionError, StreamConsumedError)
 from .utils import (
     guess_filename, get_auth_from_url, requote_uri,
     stream_decode_response_unicode, to_key_val_list, parse_header_links,
@@ -46,6 +46,8 @@ DEFAULT_REDIRECT_LIMIT = 30
 CONTENT_CHUNK_SIZE = 10 * 1024
 ITER_CHUNK_SIZE = 512
 
+json_dumps = json.dumps
+
 
 class RequestEncodingMixin(object):
     @property
@@ -189,7 +191,8 @@ class Request(RequestHooksMixin):
     :param url: URL to send.
     :param headers: dictionary of headers to send.
     :param files: dictionary of {filename: fileobject} files to multipart upload.
-    :param data: the body to attach the request. If a dictionary is provided, form-encoding will take place.
+    :param data: the body to attach to the request. If a dictionary is provided, form-encoding will take place.
+    :param json: json for the body to attach to the request (if data is not specified).
     :param params: dictionary of URL parameters to append to the URL.
     :param auth: Auth handler or (user, pass) tuple.
     :param cookies: dictionary or CookieJar of cookies to attach to this request.
@@ -212,7 +215,8 @@ class Request(RequestHooksMixin):
         params=None,
         auth=None,
         cookies=None,
-        hooks=None):
+        hooks=None,
+        json=None):
 
         # Default empty dicts for dict params.
         data = [] if data is None else data
@@ -230,6 +234,7 @@ class Request(RequestHooksMixin):
         self.headers = headers
         self.files = files
         self.data = data
+        self.json = json
         self.params = params
         self.auth = auth
         self.cookies = cookies
@@ -246,6 +251,7 @@ class Request(RequestHooksMixin):
             headers=self.headers,
             files=self.files,
             data=self.data,
+            json=self.json,
             params=self.params,
             auth=self.auth,
             cookies=self.cookies,
@@ -289,14 +295,15 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
         self.hooks = default_hooks()
 
     def prepare(self, method=None, url=None, headers=None, files=None,
-                data=None, params=None, auth=None, cookies=None, hooks=None):
+                data=None, params=None, auth=None, cookies=None, hooks=None,
+                json=None):
         """Prepares the entire request with the given parameters."""
 
         self.prepare_method(method)
         self.prepare_url(url, params)
         self.prepare_headers(headers)
         self.prepare_cookies(cookies)
-        self.prepare_body(data, files)
+        self.prepare_body(data, files, json)
         self.prepare_auth(auth, url)
         # Note that prepare_auth must be last to enable authentication schemes
         # such as OAuth to work on a fully prepared request.
@@ -326,21 +333,27 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
     def prepare_url(self, url, params):
         """Prepares the given HTTP URL."""
         #: Accept objects that have string representations.
-        try:
+        #: We're unable to blindy call unicode/str functions
-            url = unicode(url)
+        #: as this will include the bytestring indicator (b'')
-        except NameError:
+        #: on python 3.x.
-            # We're on Python 3.
+        #: https://github.com/kennethreitz/requests/pull/2238
-            url = str(url)
+        if isinstance(url, bytes):
-        except UnicodeDecodeError:
+            url = url.decode('utf8')
-            pass
+        else:
+            url = unicode(url) if is_py2 else str(url)
 
-        # Don't do any URL preparation for oddball schemes
+        # Don't do any URL preparation for non-HTTP schemes like `mailto`,
+        # `data` etc to work around exceptions from `url_parse`, which
+        # handles RFC 3986 only.
         if ':' in url and not url.lower().startswith('http'):
             self.url = url
             return
 
         # Support for unicode domain names and paths.
-        scheme, auth, host, port, path, query, fragment = parse_url(url)
+        try:
+            scheme, auth, host, port, path, query, fragment = parse_url(url)
+        except LocationParseError as e:
+            raise InvalidURL(*e.args)
 
         if not scheme:
             raise MissingSchema("Invalid URL {0!r}: No schema supplied. "
@@ -397,7 +410,7 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
         else:
             self.headers = CaseInsensitiveDict()
 
-    def prepare_body(self, data, files):
+    def prepare_body(self, data, files, json=None):
         """Prepares the given HTTP body data."""
 
         # Check if file, fo, generator, iterator.
@@ -408,6 +421,10 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
         content_type = None
         length = None
 
+        if json is not None:
+            content_type = 'application/json'
+            body = json_dumps(json)
+
         is_stream = all([
             hasattr(data, '__iter__'),
             not isinstance(data, (basestring, list, tuple, dict))
@@ -433,7 +450,7 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
             if files:
                 (body, content_type) = self._encode_files(files, data)
             else:
-                if data:
+                if data and json is None:
                     body = self._encode_params(data)
                     if isinstance(data, basestring) or hasattr(data, 'read'):
                         content_type = None
@@ -443,7 +460,7 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
             self.prepare_content_length(body)
 
             # Add content-type if it wasn't explicitly provided.
-            if (content_type) and (not 'content-type' in self.headers):
+            if content_type and ('content-type' not in self.headers):
                 self.headers['Content-Type'] = content_type
 
         self.body = body
@@ -457,7 +474,7 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
             l = super_len(body)
             if l:
                 self.headers['Content-Length'] = builtin_str(l)
-        elif self.method not in ('GET', 'HEAD'):
+        elif (self.method not in ('GET', 'HEAD')) and (self.headers.get('Content-Length') is None):
             self.headers['Content-Length'] = '0'
 
     def prepare_auth(self, auth, url=''):
@@ -600,7 +617,7 @@ class Response(object):
     def ok(self):
         try:
             self.raise_for_status()
-        except RequestException:
+        except HTTPError:
             return False
         return True
 
@@ -653,6 +670,8 @@ class Response(object):
 
             self._content_consumed = True
 
+        if self._content_consumed and isinstance(self._content, bool):
+            raise StreamConsumedError()
         # simulate reading small chunks of the content
         reused_chunks = iter_slices(self._content, chunk_size)
 
@@ -665,7 +684,7 @@ class Response(object):
 
         return chunks
 
-    def iter_lines(self, chunk_size=ITER_CHUNK_SIZE, decode_unicode=None):
+    def iter_lines(self, chunk_size=ITER_CHUNK_SIZE, decode_unicode=None, delimiter=None):
         """Iterates over the response data, one line at a time.  When
         stream=True is set on the request, this avoids reading the
         content at once into memory for large responses.
@@ -677,7 +696,11 @@ class Response(object):
 
             if pending is not None:
                 chunk = pending + chunk
-            lines = chunk.splitlines()
+
+            if delimiter:
+                lines = chunk.split(delimiter)
+            else:
+                lines = chunk.splitlines()
 
             if lines and lines[-1] and chunk and lines[-1][-1] == chunk[-1]:
                 pending = lines.pop()

libs/requests/packages/urllib3/__init__.py

@@ -57,7 +57,7 @@ del NullHandler
 
 # Set security warning to only go off once by default.
 import warnings
-warnings.simplefilter('module', exceptions.SecurityWarning)
+warnings.simplefilter('always', exceptions.SecurityWarning)
 
 def disable_warnings(category=exceptions.HTTPWarning):
     """

libs/requests/packages/urllib3/_collections.py

@@ -14,7 +14,7 @@ try: # Python 2.7+
     from collections import OrderedDict
 except ImportError:
     from .packages.ordered_dict import OrderedDict
-from .packages.six import itervalues
+from .packages.six import iterkeys, itervalues
 
 
 __all__ = ['RecentlyUsedContainer', 'HTTPHeaderDict']
@@ -85,8 +85,7 @@ class RecentlyUsedContainer(MutableMapping):
     def clear(self):
         with self.lock:
             # Copy pointers to all values, then wipe the mapping
-            # under Python 2, this copies the list of values twice :-|
+            values = list(itervalues(self._container))
-            values = list(self._container.values())
             self._container.clear()
 
         if self.dispose_func:
@@ -95,7 +94,7 @@ class RecentlyUsedContainer(MutableMapping):
 
     def keys(self):
         with self.lock:
-            return self._container.keys()
+            return list(iterkeys(self._container))
 
 
 class HTTPHeaderDict(MutableMapping):

libs/requests/packages/urllib3/connection.py

@@ -3,6 +3,7 @@ import sys
 import socket
 from socket import timeout as SocketTimeout
 import warnings
+from .packages import six
 
 try:  # Python 3
     from http.client import HTTPConnection as _HTTPConnection, HTTPException
@@ -26,12 +27,20 @@ except (ImportError, AttributeError):  # Platform-specific: No SSL.
         pass
 
 
+try:  # Python 3:
+    # Not a no-op, we're adding this to the namespace so it can be imported.
+    ConnectionError = ConnectionError
+except NameError:  # Python 2:
+    class ConnectionError(Exception):
+        pass
+
+
 from .exceptions import (
     ConnectTimeoutError,
     SystemTimeWarning,
+    SecurityWarning,
 )
 from .packages.ssl_match_hostname import match_hostname
-from .packages import six
 
 from .util.ssl_ import (
     resolve_cert_reqs,
@@ -40,8 +49,8 @@ from .util.ssl_ import (
     assert_fingerprint,
 )
 
-from .util import connection
 
+from .util import connection
 
 port_by_scheme = {
     'http': 80,
@@ -233,8 +242,15 @@ class VerifiedHTTPSConnection(HTTPSConnection):
                                self.assert_fingerprint)
         elif resolved_cert_reqs != ssl.CERT_NONE \
                 and self.assert_hostname is not False:
-            match_hostname(self.sock.getpeercert(),
+            cert = self.sock.getpeercert()
-                           self.assert_hostname or hostname)
+            if not cert.get('subjectAltName', ()):
+                warnings.warn((
+                    'Certificate has no `subjectAltName`, falling back to check for a `commonName` for now. '
+                    'This feature is being removed by major browsers and deprecated by RFC 2818. '
+                    '(See https://github.com/shazow/urllib3/issues/497 for details.)'),
+                    SecurityWarning
+                )
+            match_hostname(cert, self.assert_hostname or hostname)
 
         self.is_verified = (resolved_cert_reqs == ssl.CERT_REQUIRED
                             or self.assert_fingerprint is not None)

libs/requests/packages/urllib3/connectionpool.py

@@ -32,7 +32,7 @@ from .connection import (
     port_by_scheme,
     DummyConnection,
     HTTPConnection, HTTPSConnection, VerifiedHTTPSConnection,
-    HTTPException, BaseSSLError,
+    HTTPException, BaseSSLError, ConnectionError
 )
 from .request import RequestMethods
 from .response import HTTPResponse
@@ -278,6 +278,23 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
             # can be removed later
             return Timeout.from_float(timeout)
 
+    def _raise_timeout(self, err, url, timeout_value):
+        """Is the error actually a timeout? Will raise a ReadTimeout or pass"""
+
+        if isinstance(err, SocketTimeout):
+            raise ReadTimeoutError(self, url, "Read timed out. (read timeout=%s)" % timeout_value)
+
+        # See the above comment about EAGAIN in Python 3. In Python 2 we have
+        # to specifically catch it and throw the timeout error
+        if hasattr(err, 'errno') and err.errno in _blocking_errnos:
+            raise ReadTimeoutError(self, url, "Read timed out. (read timeout=%s)" % timeout_value)
+
+        # Catch possible read timeouts thrown as SSL errors. If not the
+        # case, rethrow the original. We need to do this because of:
+        # http://bugs.python.org/issue10272
+        if 'timed out' in str(err) or 'did not complete (read)' in str(err):  # Python 2.6
+            raise ReadTimeoutError(self, url, "Read timed out. (read timeout=%s)" % timeout_value)
+
     def _make_request(self, conn, method, url, timeout=_Default,
                       **httplib_request_kw):
         """
@@ -301,7 +318,12 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
         conn.timeout = timeout_obj.connect_timeout
 
         # Trigger any extra validation we need to do.
-        self._validate_conn(conn)
+        try:
+            self._validate_conn(conn)
+        except (SocketTimeout, BaseSSLError) as e:
+            # Py2 raises this as a BaseSSLError, Py3 raises it as socket timeout.
+            self._raise_timeout(err=e, url=url, timeout_value=conn.timeout)
+            raise
 
         # conn.request() calls httplib.*.request, not the method in
         # urllib3.request. It also calls makefile (recv) on the socket.
@@ -331,28 +353,8 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
                 httplib_response = conn.getresponse(buffering=True)
             except TypeError:  # Python 2.6 and older
                 httplib_response = conn.getresponse()
-        except SocketTimeout:
+        except (SocketTimeout, BaseSSLError, SocketError) as e:
-            raise ReadTimeoutError(
+            self._raise_timeout(err=e, url=url, timeout_value=read_timeout)
-                self, url, "Read timed out. (read timeout=%s)" % read_timeout)
-
-        except BaseSSLError as e:
-            # Catch possible read timeouts thrown as SSL errors. If not the
-            # case, rethrow the original. We need to do this because of:
-            # http://bugs.python.org/issue10272
-            if 'timed out' in str(e) or \
-               'did not complete (read)' in str(e):  # Python 2.6
-                raise ReadTimeoutError(
-                        self, url, "Read timed out. (read timeout=%s)" % read_timeout)
-
-            raise
-
-        except SocketError as e:  # Platform-specific: Python 2
-            # See the above comment about EAGAIN in Python 3. In Python 2 we
-            # have to specifically catch it and throw the timeout error
-            if e.errno in _blocking_errnos:
-                raise ReadTimeoutError(
-                    self, url, "Read timed out. (read timeout=%s)" % read_timeout)
-
             raise
 
         # AppEngine doesn't have a version attr.
@@ -537,12 +539,15 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
             raise EmptyPoolError(self, "No pool connections are available.")
 
         except (BaseSSLError, CertificateError) as e:
-            # Release connection unconditionally because there is no way to
+            # Close the connection. If a connection is reused on which there
-            # close it externally in case of exception.
+            # was a Certificate error, the next request will certainly raise
-            release_conn = True
+            # another Certificate error.
+            if conn:
+                conn.close()
+                conn = None
             raise SSLError(e)
 
-        except (TimeoutError, HTTPException, SocketError) as e:
+        except (TimeoutError, HTTPException, SocketError, ConnectionError) as e:
             if conn:
                 # Discard the connection for these exceptions. It will be
                 # be replaced during the next _get_conn() call.
@@ -725,8 +730,7 @@ class HTTPSConnectionPool(HTTPConnectionPool):
             warnings.warn((
                 'Unverified HTTPS request is being made. '
                 'Adding certificate verification is strongly advised. See: '
-                'https://urllib3.readthedocs.org/en/latest/security.html '
+                'https://urllib3.readthedocs.org/en/latest/security.html'),
-                '(This warning will only appear once by default.)'),
                 InsecureRequestWarning)
 
 

libs/requests/packages/urllib3/contrib/pyopenssl.py

@@ -29,7 +29,7 @@ Now you can use :mod:`urllib3` as you normally would, and it will support SNI
 when the required modules are installed.
 
 Activating this module also has the positive side effect of disabling SSL/TLS
-encryption in Python 2 (see `CRIME attack`_).
+compression in Python 2 (see `CRIME attack`_).
 
 If you want to configure the default list of supported cipher suites, you can
 set the ``urllib3.contrib.pyopenssl.DEFAULT_SSL_CIPHER_LIST`` variable.
@@ -70,9 +70,14 @@ HAS_SNI = SUBJ_ALT_NAME_SUPPORT
 # Map from urllib3 to PyOpenSSL compatible parameter-values.
 _openssl_versions = {
     ssl.PROTOCOL_SSLv23: OpenSSL.SSL.SSLv23_METHOD,
-    ssl.PROTOCOL_SSLv3: OpenSSL.SSL.SSLv3_METHOD,
     ssl.PROTOCOL_TLSv1: OpenSSL.SSL.TLSv1_METHOD,
 }
+
+try:
+    _openssl_versions.update({ssl.PROTOCOL_SSLv3: OpenSSL.SSL.SSLv3_METHOD})
+except AttributeError:
+    pass
+
 _openssl_verify = {
     ssl.CERT_NONE: OpenSSL.SSL.VERIFY_NONE,
     ssl.CERT_OPTIONAL: OpenSSL.SSL.VERIFY_PEER,
@@ -199,8 +204,21 @@ class WrappedSocket(object):
     def settimeout(self, timeout):
         return self.socket.settimeout(timeout)
 
+    def _send_until_done(self, data):
+        while True:
+            try:
+                return self.connection.send(data)
+            except OpenSSL.SSL.WantWriteError:
+                _, wlist, _ = select.select([], [self.socket], [],
+                                            self.socket.gettimeout())
+                if not wlist:
+                    raise timeout()
+                continue
+
     def sendall(self, data):
-        return self.connection.sendall(data)
+        while len(data):
+            sent = self._send_until_done(data)
+            data = data[sent:]
 
     def close(self):
         if self._makefile_refs < 1:
@@ -248,6 +266,7 @@ def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None,
                     ssl_version=None):
     ctx = OpenSSL.SSL.Context(_openssl_versions[ssl_version])
     if certfile:
+        keyfile = keyfile or certfile  # Match behaviour of the normal python ssl library
         ctx.use_certificate_file(certfile)
     if keyfile:
         ctx.use_privatekey_file(keyfile)

libs/requests/packages/urllib3/exceptions.py

@@ -72,11 +72,8 @@ class MaxRetryError(RequestError):
     def __init__(self, pool, url, reason=None):
         self.reason = reason
 
-        message = "Max retries exceeded with url: %s" % url
+        message = "Max retries exceeded with url: %s (Caused by %r)" % (
-        if reason:
+            url, reason)
-            message += " (Caused by %r)" % reason
-        else:
-            message += " (Caused by redirect)"
 
         RequestError.__init__(self, pool, url, message)
 
@@ -141,6 +138,12 @@ class LocationParseError(LocationValueError):
         self.location = location
 
 
+class ResponseError(HTTPError):
+    "Used as a container for an error reason supplied in a MaxRetryError."
+    GENERIC_ERROR = 'too many error responses'
+    SPECIFIC_ERROR = 'too many {status_code} error responses'
+
+
 class SecurityWarning(HTTPWarning):
     "Warned when perfoming security reducing actions"
     pass

libs/requests/packages/urllib3/request.py

@@ -118,18 +118,24 @@ class RequestMethods(object):
         which is used to compose the body of the request. The random boundary
         string can be explicitly set with the ``multipart_boundary`` parameter.
         """
-        if encode_multipart:
-            body, content_type = encode_multipart_formdata(
-                fields or {}, boundary=multipart_boundary)
-        else:
-            body, content_type = (urlencode(fields or {}),
-                                  'application/x-www-form-urlencoded')
-
         if headers is None:
             headers = self.headers
 
-        headers_ = {'Content-Type': content_type}
+        extra_kw = {'headers': {}}
-        headers_.update(headers)
 
-        return self.urlopen(method, url, body=body, headers=headers_,
+        if fields:
-                            **urlopen_kw)
+            if 'body' in urlopen_kw:
+                raise TypeError('request got values for both \'fields\' and \'body\', can only specify one.')
+
+            if encode_multipart:
+                body, content_type = encode_multipart_formdata(fields, boundary=multipart_boundary)
+            else:
+                body, content_type = urlencode(fields), 'application/x-www-form-urlencoded'
+
+            extra_kw['body'] = body
+            extra_kw['headers'] = {'Content-Type': content_type}
+
+        extra_kw['headers'].update(headers)
+        extra_kw.update(urlopen_kw)
+
+        return self.urlopen(method, url, **extra_kw)

libs/requests/packages/urllib3/util/retry.py

@@ -2,10 +2,11 @@ import time
 import logging
 
 from ..exceptions import (
-    ProtocolError,
     ConnectTimeoutError,
-    ReadTimeoutError,
     MaxRetryError,
+    ProtocolError,
+    ReadTimeoutError,
+    ResponseError,
 )
 from ..packages import six
 
@@ -36,7 +37,6 @@ class Retry(object):
     Errors will be wrapped in :class:`~urllib3.exceptions.MaxRetryError` unless
     retries are disabled, in which case the causing exception will be raised.
 
-
     :param int total:
         Total number of retries to allow. Takes precedence over other counts.
 
@@ -184,8 +184,8 @@ class Retry(object):
         return isinstance(err, ConnectTimeoutError)
 
     def _is_read_error(self, err):
-        """ Errors that occur after the request has been started, so we can't
+        """ Errors that occur after the request has been started, so we should
-        assume that the server did not process any of it.
+        assume that the server began processing it.
         """
         return isinstance(err, (ReadTimeoutError, ProtocolError))
 
@@ -198,8 +198,7 @@ class Retry(object):
         return self.status_forcelist and status_code in self.status_forcelist
 
     def is_exhausted(self):
-        """ Are we out of retries?
+        """ Are we out of retries? """
-        """
         retry_counts = (self.total, self.connect, self.read, self.redirect)
         retry_counts = list(filter(None, retry_counts))
         if not retry_counts:
@@ -230,6 +229,7 @@ class Retry(object):
         connect = self.connect
         read = self.read
         redirect = self.redirect
+        cause = 'unknown'
 
         if error and self._is_connection_error(error):
             # Connect retry?
@@ -251,10 +251,16 @@ class Retry(object):
             # Redirect retry?
             if redirect is not None:
                 redirect -= 1
+            cause = 'too many redirects'
 
         else:
-            # FIXME: Nothing changed, scenario doesn't make sense.
+            # Incrementing because of a server error like a 500 in
+            # status_forcelist and a the given method is in the whitelist
             _observed_errors += 1
+            cause = ResponseError.GENERIC_ERROR
+            if response and response.status:
+                cause = ResponseError.SPECIFIC_ERROR.format(
+                    status_code=response.status)
 
         new_retry = self.new(
             total=total,
@@ -262,7 +268,7 @@ class Retry(object):
             _observed_errors=_observed_errors)
 
         if new_retry.is_exhausted():
-            raise MaxRetryError(_pool, url, error)
+            raise MaxRetryError(_pool, url, error or ResponseError(cause))
 
         log.debug("Incremented Retry for (url='%s'): %r" % (url, new_retry))
 

libs/requests/packages/urllib3/util/ssl_.py

@@ -4,18 +4,84 @@ from hashlib import md5, sha1
 from ..exceptions import SSLError
 
 
-try:  # Test for SSL features
+SSLContext = None
-    SSLContext = None
+HAS_SNI = False
-    HAS_SNI = False
+create_default_context = None
 
-    import ssl
+import errno
+import ssl
+
+try:  # Test for SSL features
     from ssl import wrap_socket, CERT_NONE, PROTOCOL_SSLv23
-    from ssl import SSLContext  # Modern SSL?
     from ssl import HAS_SNI  # Has SNI?
 except ImportError:
     pass
 
 
+try:
+    from ssl import OP_NO_SSLv2, OP_NO_SSLv3, OP_NO_COMPRESSION
+except ImportError:
+    OP_NO_SSLv2, OP_NO_SSLv3 = 0x1000000, 0x2000000
+    OP_NO_COMPRESSION = 0x20000
+
+try:
+    from ssl import _DEFAULT_CIPHERS
+except ImportError:
+    _DEFAULT_CIPHERS = (
+        'ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+HIGH:'
+        'DH+HIGH:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+HIGH:RSA+3DES:ECDH+RC4:'
+        'DH+RC4:RSA+RC4:!aNULL:!eNULL:!MD5'
+    )
+
+try:
+    from ssl import SSLContext  # Modern SSL?
+except ImportError:
+    import sys
+
+    class SSLContext(object):  # Platform-specific: Python 2 & 3.1
+        supports_set_ciphers = sys.version_info >= (2, 7)
+
+        def __init__(self, protocol_version):
+            self.protocol = protocol_version
+            # Use default values from a real SSLContext
+            self.check_hostname = False
+            self.verify_mode = ssl.CERT_NONE
+            self.ca_certs = None
+            self.options = 0
+            self.certfile = None
+            self.keyfile = None
+            self.ciphers = None
+
+        def load_cert_chain(self, certfile, keyfile):
+            self.certfile = certfile
+            self.keyfile = keyfile
+
+        def load_verify_locations(self, location):
+            self.ca_certs = location
+
+        def set_ciphers(self, cipher_suite):
+            if not self.supports_set_ciphers:
+                raise TypeError(
+                    'Your version of Python does not support setting '
+                    'a custom cipher suite. Please upgrade to Python '
+                    '2.7, 3.2, or later if you need this functionality.'
+                )
+            self.ciphers = cipher_suite
+
+        def wrap_socket(self, socket, server_hostname=None):
+            kwargs = {
+                'keyfile': self.keyfile,
+                'certfile': self.certfile,
+                'ca_certs': self.ca_certs,
+                'cert_reqs': self.verify_mode,
+                'ssl_version': self.protocol,
+            }
+            if self.supports_set_ciphers:  # Platform-specific: Python 2.7+
+                return wrap_socket(socket, ciphers=self.ciphers, **kwargs)
+            else:  # Platform-specific: Python 2.6
+                return wrap_socket(socket, **kwargs)
+
+
 def assert_fingerprint(cert, fingerprint):
     """
     Checks if given fingerprint matches the supplied certificate.
@@ -91,42 +157,98 @@ def resolve_ssl_version(candidate):
     return candidate
 
 
-if SSLContext is not None:  # Python 3.2+
+def create_urllib3_context(ssl_version=None, cert_reqs=ssl.CERT_REQUIRED,
-    def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None,
+                           options=None, ciphers=None):
-                        ca_certs=None, server_hostname=None,
+    """All arguments have the same meaning as ``ssl_wrap_socket``.
-                        ssl_version=None):
-        """
-        All arguments except `server_hostname` have the same meaning as for
-        :func:`ssl.wrap_socket`
 
-        :param server_hostname:
+    By default, this function does a lot of the same work that
-            Hostname of the expected certificate
+    ``ssl.create_default_context`` does on Python 3.4+. It:
-        """
-        context = SSLContext(ssl_version)
-        context.verify_mode = cert_reqs
 
-        # Disable TLS compression to migitate CRIME attack (issue #309)
+    - Disables SSLv2, SSLv3, and compression
-        OP_NO_COMPRESSION = 0x20000
+    - Sets a restricted set of server ciphers
-        context.options |= OP_NO_COMPRESSION
 
-        if ca_certs:
+    If you wish to enable SSLv3, you can do::
-            try:
+
-                context.load_verify_locations(ca_certs)
+        from urllib3.util import ssl_
-            # Py32 raises IOError
+        context = ssl_.create_urllib3_context()
-            # Py33 raises FileNotFoundError
+        context.options &= ~ssl_.OP_NO_SSLv3
-            except Exception as e:  # Reraise as SSLError
+
+    You can do the same to enable compression (substituting ``COMPRESSION``
+    for ``SSLv3`` in the last line above).
+
+    :param ssl_version:
+        The desired protocol version to use. This will default to
+        PROTOCOL_SSLv23 which will negotiate the highest protocol that both
+        the server and your installation of OpenSSL support.
+    :param cert_reqs:
+        Whether to require the certificate verification. This defaults to
+        ``ssl.CERT_REQUIRED``.
+    :param options:
+        Specific OpenSSL options. These default to ``ssl.OP_NO_SSLv2``,
+        ``ssl.OP_NO_SSLv3``, ``ssl.OP_NO_COMPRESSION``.
+    :param ciphers:
+        Which cipher suites to allow the server to select.
+    :returns:
+        Constructed SSLContext object with specified options
+    :rtype: SSLContext
+    """
+    context = SSLContext(ssl_version or ssl.PROTOCOL_SSLv23)
+
+    if options is None:
+        options = 0
+        # SSLv2 is easily broken and is considered harmful and dangerous
+        options |= OP_NO_SSLv2
+        # SSLv3 has several problems and is now dangerous
+        options |= OP_NO_SSLv3
+        # Disable compression to prevent CRIME attacks for OpenSSL 1.0+
+        # (issue #309)
+        options |= OP_NO_COMPRESSION
+
+    context.options |= options
+
+    if getattr(context, 'supports_set_ciphers', True):  # Platform-specific: Python 2.6
+        context.set_ciphers(ciphers or _DEFAULT_CIPHERS)
+
+    context.verify_mode = cert_reqs
+    if getattr(context, 'check_hostname', None) is not None:  # Platform-specific: Python 3.2
+        context.check_hostname = (context.verify_mode == ssl.CERT_REQUIRED)
+    return context
+
+
+def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None,
+                    ca_certs=None, server_hostname=None,
+                    ssl_version=None, ciphers=None, ssl_context=None):
+    """
+    All arguments except for server_hostname and ssl_context have the same
+    meaning as they do when using :func:`ssl.wrap_socket`.
+
+    :param server_hostname:
+        When SNI is supported, the expected hostname of the certificate
+    :param ssl_context:
+        A pre-made :class:`SSLContext` object. If none is provided, one will
+        be created using :func:`create_urllib3_context`.
+    :param ciphers:
+        A string of ciphers we wish the client to support. This is not
+        supported on Python 2.6 as the ssl module does not support it.
+    """
+    context = ssl_context
+    if context is None:
+        context = create_urllib3_context(ssl_version, cert_reqs,
+                                         ciphers=ciphers)
+
+    if ca_certs:
+        try:
+            context.load_verify_locations(ca_certs)
+        except IOError as e:  # Platform-specific: Python 2.6, 2.7, 3.2
+            raise SSLError(e)
+        # Py33 raises FileNotFoundError which subclasses OSError
+        # These are not equivalent unless we check the errno attribute
+        except OSError as e:  # Platform-specific: Python 3.3 and beyond
+            if e.errno == errno.ENOENT:
                 raise SSLError(e)
-        if certfile:
+            raise
-            # FIXME: This block needs a test.
+    if certfile:
-            context.load_cert_chain(certfile, keyfile)
+        context.load_cert_chain(certfile, keyfile)
-        if HAS_SNI:  # Platform-specific: OpenSSL with enabled SNI
+    if HAS_SNI:  # Platform-specific: OpenSSL with enabled SNI
-            return context.wrap_socket(sock, server_hostname=server_hostname)
+        return context.wrap_socket(sock, server_hostname=server_hostname)
-        return context.wrap_socket(sock)
+    return context.wrap_socket(sock)
-
-else:  # Python 3.1 and earlier
-    def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None,
-                        ca_certs=None, server_hostname=None,
-                        ssl_version=None):
-        return wrap_socket(sock, keyfile=keyfile, certfile=certfile,
-                           ca_certs=ca_certs, cert_reqs=cert_reqs,
-                           ssl_version=ssl_version)

libs/requests/packages/urllib3/util/url.py

@@ -40,6 +40,48 @@ class Url(namedtuple('Url', url_attrs)):
             return '%s:%d' % (self.host, self.port)
         return self.host
 
+    @property
+    def url(self):
+        """
+        Convert self into a url
+
+        This function should more or less round-trip with :func:`.parse_url`. The
+        returned url may not be exactly the same as the url inputted to
+        :func:`.parse_url`, but it should be equivalent by the RFC (e.g., urls
+        with a blank port will have : removed).
+
+        Example: ::
+
+            >>> U = parse_url('http://google.com/mail/')
+            >>> U.url
+            'http://google.com/mail/'
+            >>> Url('http', 'username:password', 'host.com', 80,
+            ... '/path', 'query', 'fragment').url
+            'http://username:password@host.com:80/path?query#fragment'
+        """
+        scheme, auth, host, port, path, query, fragment = self
+        url = ''
+
+        # We use "is not None" we want things to happen with empty strings (or 0 port)
+        if scheme is not None:
+            url += scheme + '://'
+        if auth is not None:
+            url += auth + '@'
+        if host is not None:
+            url += host
+        if port is not None:
+            url += ':' + str(port)
+        if path is not None:
+            url += path
+        if query is not None:
+            url += '?' + query
+        if fragment is not None:
+            url += '#' + fragment
+
+        return url
+
+    def __str__(self):
+        return self.url
 
 def split_first(s, delims):
     """
@@ -84,7 +126,7 @@ def parse_url(url):
     Example::
 
         >>> parse_url('http://google.com/mail/')
-        Url(scheme='http', host='google.com', port=None, path='/', ...)
+        Url(scheme='http', host='google.com', port=None, path='/mail/', ...)
         >>> parse_url('google.com:80')
         Url(scheme=None, host='google.com', port=80, path=None, ...)
         >>> parse_url('/foo?bar')
@@ -162,7 +204,6 @@ def parse_url(url):
 
     return Url(scheme, auth, host, port, path, query, fragment)
 
-
 def get_host(url):
     """
     Deprecated. Use :func:`.parse_url` instead.

libs/requests/sessions.py

@@ -13,7 +13,7 @@ from collections import Mapping
 from datetime import datetime
 
 from .auth import _basic_auth_str
-from .compat import cookielib, OrderedDict, urljoin, urlparse, builtin_str
+from .compat import cookielib, OrderedDict, urljoin, urlparse
 from .cookies import (
     cookiejar_from_dict, extract_cookies_to_jar, RequestsCookieJar, merge_cookies)
 from .models import Request, PreparedRequest, DEFAULT_REDIRECT_LIMIT
@@ -21,6 +21,7 @@ from .hooks import default_hooks, dispatch_hook
 from .utils import to_key_val_list, default_headers, to_native_string
 from .exceptions import (
     TooManyRedirects, InvalidSchema, ChunkedEncodingError, ContentDecodingError)
+from .packages.urllib3._collections import RecentlyUsedContainer
 from .structures import CaseInsensitiveDict
 
 from .adapters import HTTPAdapter
@@ -35,6 +36,8 @@ from .status_codes import codes
 # formerly defined here, reexposed here for backward compatibility
 from .models import REDIRECT_STATI
 
+REDIRECT_CACHE_SIZE = 1000
+
 
 def merge_setting(request_setting, session_setting, dict_class=OrderedDict):
     """
@@ -128,14 +131,14 @@ class SessionRedirectMixin(object):
             # Facilitate relative 'location' headers, as allowed by RFC 7231.
             # (e.g. '/path/to/resource' instead of 'http://domain.tld/path/to/resource')
             # Compliant with RFC3986, we percent encode the url.
-            if not urlparse(url).netloc:
+            if not parsed.netloc:
                 url = urljoin(resp.url, requote_uri(url))
             else:
                 url = requote_uri(url)
 
             prepared_request.url = to_native_string(url)
-            # cache the url
+            # Cache the url, unless it redirects to itself.
-            if resp.is_permanent_redirect:
+            if resp.is_permanent_redirect and req.url != prepared_request.url:
                 self.redirect_cache[req.url] = prepared_request.url
 
             # http://tools.ietf.org/html/rfc7231#section-6.4.4
@@ -271,9 +274,10 @@ class Session(SessionRedirectMixin):
     """
 
     __attrs__ = [
-        'headers', 'cookies', 'auth', 'timeout', 'proxies', 'hooks',
+        'headers', 'cookies', 'auth', 'proxies', 'hooks', 'params', 'verify',
-        'params', 'verify', 'cert', 'prefetch', 'adapters', 'stream',
+        'cert', 'prefetch', 'adapters', 'stream', 'trust_env',
-        'trust_env', 'max_redirects', 'redirect_cache']
+        'max_redirects',
+    ]
 
     def __init__(self):
 
@@ -326,7 +330,8 @@ class Session(SessionRedirectMixin):
         self.mount('https://', HTTPAdapter())
         self.mount('http://', HTTPAdapter())
 
-        self.redirect_cache = {}
+        # Only store 1000 redirects to prevent using infinite memory
+        self.redirect_cache = RecentlyUsedContainer(REDIRECT_CACHE_SIZE)
 
     def __enter__(self):
         return self
@@ -365,6 +370,7 @@ class Session(SessionRedirectMixin):
             url=request.url,
             files=request.files,
             data=request.data,
+            json=request.json,
             headers=merge_setting(request.headers, self.headers, dict_class=CaseInsensitiveDict),
             params=merge_setting(request.params, self.params),
             auth=merge_setting(auth, self.auth),
@@ -386,7 +392,8 @@ class Session(SessionRedirectMixin):
         hooks=None,
         stream=None,
         verify=None,
-        cert=None):
+        cert=None,
+        json=None):
         """Constructs a :class:`Request <Request>`, prepares it and sends it.
         Returns :class:`Response <Response>` object.
 
@@ -396,17 +403,22 @@ class Session(SessionRedirectMixin):
             string for the :class:`Request`.
         :param data: (optional) Dictionary or bytes to send in the body of the
             :class:`Request`.
+        :param json: (optional) json to send in the body of the
+            :class:`Request`.
         :param headers: (optional) Dictionary of HTTP Headers to send with the
             :class:`Request`.
         :param cookies: (optional) Dict or CookieJar object to send with the
             :class:`Request`.
-        :param files: (optional) Dictionary of 'filename': file-like-objects
+        :param files: (optional) Dictionary of ``'filename': file-like-objects``
             for multipart encoding upload.
         :param auth: (optional) Auth tuple or callable to enable
             Basic/Digest/Custom HTTP Auth.
-        :param timeout: (optional) Float describing the timeout of the
+        :param timeout: (optional) How long to wait for the server to send
-            request in seconds.
+            data before giving up, as a float, or a (`connect timeout, read
-        :param allow_redirects: (optional) Boolean. Set to True by default.
+            timeout <user/advanced.html#timeouts>`_) tuple.
+        :type timeout: float or tuple
+        :param allow_redirects: (optional) Set to True by default.
+        :type allow_redirects: bool
         :param proxies: (optional) Dictionary mapping protocol to the URL of
             the proxy.
         :param stream: (optional) whether to immediately download the response
@@ -417,7 +429,7 @@ class Session(SessionRedirectMixin):
             If Tuple, ('cert', 'key') pair.
         """
 
-        method = builtin_str(method)
+        method = to_native_string(method)
 
         # Create the Request.
         req = Request(
@@ -426,6 +438,7 @@ class Session(SessionRedirectMixin):
             headers = headers,
             files = files,
             data = data or {},
+            json = json,
             params = params or {},
             auth = auth,
             cookies = cookies,
@@ -479,15 +492,16 @@ class Session(SessionRedirectMixin):
         kwargs.setdefault('allow_redirects', False)
         return self.request('HEAD', url, **kwargs)
 
-    def post(self, url, data=None, **kwargs):
+    def post(self, url, data=None, json=None, **kwargs):
         """Sends a POST request. Returns :class:`Response` object.
 
         :param url: URL for the new :class:`Request` object.
         :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`.
+        :param json: (optional) json to send in the body of the :class:`Request`.
         :param \*\*kwargs: Optional arguments that ``request`` takes.
         """
 
-        return self.request('POST', url, data=data, **kwargs)
+        return self.request('POST', url, data=data, json=json, **kwargs)
 
     def put(self, url, data=None, **kwargs):
         """Sends a PUT request. Returns :class:`Response` object.
@@ -532,12 +546,13 @@ class Session(SessionRedirectMixin):
         if not isinstance(request, PreparedRequest):
             raise ValueError('You can only send PreparedRequests.')
 
-        redirect_count = 0
+        checked_urls = set()
         while request.url in self.redirect_cache:
-            redirect_count += 1
+            checked_urls.add(request.url)
-            if redirect_count > self.max_redirects:
+            new_url = self.redirect_cache.get(request.url)
-                raise TooManyRedirects
+            if new_url in checked_urls:
-            request.url = self.redirect_cache.get(request.url)
+                break
+            request.url = new_url
 
         # Set up variables needed for resolve_redirects and dispatching of hooks
         allow_redirects = kwargs.pop('allow_redirects', True)
@@ -647,12 +662,19 @@ class Session(SessionRedirectMixin):
             self.adapters[key] = self.adapters.pop(key)
 
     def __getstate__(self):
-        return dict((attr, getattr(self, attr, None)) for attr in self.__attrs__)
+        state = dict((attr, getattr(self, attr, None)) for attr in self.__attrs__)
+        state['redirect_cache'] = dict(self.redirect_cache)
+        return state
 
     def __setstate__(self, state):
+        redirect_cache = state.pop('redirect_cache', {})
         for attr, value in state.items():
             setattr(self, attr, value)
 
+        self.redirect_cache = RecentlyUsedContainer(REDIRECT_CACHE_SIZE)
+        for redirect, to in redirect_cache.items():
+            self.redirect_cache[redirect] = to
+
 
 def session():
     """Returns a :class:`Session` for context-management."""

libs/requests/utils.py

@@ -19,6 +19,7 @@ import re
 import sys
 import socket
 import struct
+import warnings
 
 from . import __version__
 from . import certs
@@ -114,7 +115,7 @@ def get_netrc_auth(url):
 def guess_filename(obj):
     """Tries to guess the filename of the given object."""
     name = getattr(obj, 'name', None)
-    if name and name[0] != '<' and name[-1] != '>':
+    if name and isinstance(name, builtin_str) and name[0] != '<' and name[-1] != '>':
         return os.path.basename(name)
 
 
@@ -287,6 +288,11 @@ def get_encodings_from_content(content):
 
     :param content: bytestring to extract encodings from.
     """
+    warnings.warn((
+        'In requests 3.0, get_encodings_from_content will be removed. For '
+        'more information, please see the discussion on issue #2266. (This'
+        ' warning should only appear once.)'),
+        DeprecationWarning)
 
     charset_re = re.compile(r'<meta.*?charset=["\']*(.+?)["\'>]', flags=re.I)
     pragma_re = re.compile(r'<meta.*?content=["\']*;?charset=(.+?)["\'>]', flags=re.I)
@@ -351,12 +357,14 @@ def get_unicode_from_response(r):
     Tried:
 
     1. charset from content-type
-
+    2. fall back and replace all unicode characters
-    2. every encodings from ``<meta ... charset=XXX>``
-
-    3. fall back and replace all unicode characters
 
     """
+    warnings.warn((
+        'In requests 3.0, get_unicode_from_response will be removed. For '
+        'more information, please see the discussion on issue #2266. (This'
+        ' warning should only appear once.)'),
+        DeprecationWarning)
 
     tried_encodings = []
 
@@ -555,7 +563,7 @@ def default_headers():
         'User-Agent': default_user_agent(),
         'Accept-Encoding': ', '.join(('gzip', 'deflate')),
         'Accept': '*/*',
-        'Connection': 'keep-alive'
+        'Connection': 'keep-alive',
     })
 
 
@@ -570,7 +578,7 @@ def parse_header_links(value):
 
     replace_chars = " '\""
 
-    for val in value.split(","):
+    for val in re.split(", *<", value):
         try:
             url, params = val.split(";", 1)
         except ValueError:
@@ -672,3 +680,18 @@ def to_native_string(string, encoding='ascii'):
             out = string.decode(encoding)
 
     return out
+
+
+def urldefragauth(url):
+    """
+    Given a url remove the fragment and the authentication part
+    """
+    scheme, netloc, path, params, query, fragment = urlparse(url)
+
+    # see func:`prepend_scheme_if_needed`
+    if not netloc:
+        netloc, path = path, netloc
+
+    netloc = netloc.rsplit('@', 1)[-1]
+
+    return urlunparse((scheme, netloc, path, params, query, ''))

libs/tornado/httpclient.py

@@ -95,7 +95,8 @@ class HTTPClient(object):
         If it is a string, we construct an `HTTPRequest` using any additional
         kwargs: ``HTTPRequest(request, **kwargs)``
 
-        If an error occurs during the fetch, we raise an `HTTPError`.
+        If an error occurs during the fetch, we raise an `HTTPError` unless
+        the ``raise_error`` keyword argument is set to False.
         """
         response = self._io_loop.run_sync(functools.partial(
             self._async_client.fetch, request, **kwargs))
@@ -200,7 +201,7 @@ class AsyncHTTPClient(Configurable):
                 raise RuntimeError("inconsistent AsyncHTTPClient cache")
             del self._instance_cache[self.io_loop]
 
-    def fetch(self, request, callback=None, **kwargs):
+    def fetch(self, request, callback=None, raise_error=True, **kwargs):
         """Executes a request, asynchronously returning an `HTTPResponse`.
 
         The request may be either a string URL or an `HTTPRequest` object.
@@ -208,8 +209,10 @@ class AsyncHTTPClient(Configurable):
         kwargs: ``HTTPRequest(request, **kwargs)``
 
         This method returns a `.Future` whose result is an
-        `HTTPResponse`.  The ``Future`` will raise an `HTTPError` if
+        `HTTPResponse`.  By default, the ``Future`` will raise an `HTTPError`
-        the request returned a non-200 response code.
+        if the request returned a non-200 response code. Instead, if
+        ``raise_error`` is set to False, the response will always be
+        returned regardless of the response code.
 
         If a ``callback`` is given, it will be invoked with the `HTTPResponse`.
         In the callback interface, `HTTPError` is not automatically raised.
@@ -243,7 +246,7 @@ class AsyncHTTPClient(Configurable):
             future.add_done_callback(handle_future)
 
         def handle_response(response):
-            if response.error:
+            if raise_error and response.error:
                 future.set_exception(response.error)
             else:
                 future.set_result(response)

libs/tornado/httpserver.py

@@ -42,30 +42,10 @@ from tornado.tcpserver import TCPServer
 class HTTPServer(TCPServer, httputil.HTTPServerConnectionDelegate):
     r"""A non-blocking, single-threaded HTTP server.
 
-    A server is defined by either a request callback that takes a
+    A server is defined by a subclass of `.HTTPServerConnectionDelegate`,
-    `.HTTPServerRequest` as an argument or a `.HTTPServerConnectionDelegate`
+    or, for backwards compatibility, a callback that takes an
-    instance.
+    `.HTTPServerRequest` as an argument. The delegate is usually a
-
+    `tornado.web.Application`.
-    A simple example server that echoes back the URI you requested::
-
-        import tornado.httpserver
-        import tornado.ioloop
-        from tornado import httputil
-
-        def handle_request(request):
-           message = "You requested %s\n" % request.uri
-           request.connection.write_headers(
-               httputil.ResponseStartLine('HTTP/1.1', 200, 'OK'),
-               httputil.HTTPHeaders({"Content-Length": str(len(message))}))
-           request.connection.write(message)
-           request.connection.finish()
-
-        http_server = tornado.httpserver.HTTPServer(handle_request)
-        http_server.listen(8888)
-        tornado.ioloop.IOLoop.instance().start()
-
-    Applications should use the methods of `.HTTPConnection` to write
-    their response.
 
     `HTTPServer` supports keep-alive connections by default
     (automatically for HTTP/1.1, or for HTTP/1.0 when the client

libs/tornado/httputil.py

@@ -331,7 +331,7 @@ class HTTPServerRequest(object):
         self.uri = uri
         self.version = version
         self.headers = headers or HTTPHeaders()
-        self.body = body or ""
+        self.body = body or b""
 
         # set remote IP and protocol
         context = getattr(connection, 'context', None)
@@ -873,3 +873,17 @@ def _encode_header(key, pdict):
 def doctests():
     import doctest
     return doctest.DocTestSuite()
+
+def split_host_and_port(netloc):
+    """Returns ``(host, port)`` tuple from ``netloc``.
+
+    Returned ``port`` will be ``None`` if not present.
+    """
+    match = re.match(r'^(.+):(\d+)$', netloc)
+    if match:
+        host = match.group(1)
+        port = int(match.group(2))
+    else:
+        host = netloc
+        port = None
+    return (host, port)

libs/tornado/iostream.py

@@ -331,7 +331,7 @@ class BaseIOStream(object):
         if data:
             if (self.max_write_buffer_size is not None and
                     self._write_buffer_size + len(data) > self.max_write_buffer_size):
-                raise StreamBufferFullError("Reached maximum read buffer size")
+                raise StreamBufferFullError("Reached maximum write buffer size")
             # Break up large contiguous strings before inserting them in the
             # write buffer, so we don't have to recopy the entire thing
             # as we slice off pieces to send to the socket.

libs/tornado/netutil.py

@@ -20,7 +20,7 @@ from __future__ import absolute_import, division, print_function, with_statement
 
 import errno
 import os
-import platform
+import sys
 import socket
 import stat
 
@@ -105,7 +105,7 @@ def bind_sockets(port, address=None, family=socket.AF_UNSPEC,
     for res in set(socket.getaddrinfo(address, port, family, socket.SOCK_STREAM,
                                       0, flags)):
         af, socktype, proto, canonname, sockaddr = res
-        if (platform.system() == 'Darwin' and address == 'localhost' and
+        if (sys.platform == 'darwin' and address == 'localhost' and
                 af == socket.AF_INET6 and sockaddr[3] != 0):
             # Mac OS X includes a link-local address fe80::1%lo0 in the
             # getaddrinfo results for 'localhost'.  However, the firewall

libs/tornado/options.py

@@ -204,6 +204,13 @@ class OptionParser(object):
                         (name, self._options[name].file_name))
         frame = sys._getframe(0)
         options_file = frame.f_code.co_filename
+
+        # Can be called directly, or through top level define() fn, in which
+        # case, step up above that frame to look for real caller.
+        if (frame.f_back.f_code.co_filename == options_file and
+                frame.f_back.f_code.co_name == 'define'):
+            frame = frame.f_back
+
         file_name = frame.f_back.f_code.co_filename
         if file_name == options_file:
             file_name = ""

libs/tornado/platform/kqueue.py

@@ -54,8 +54,7 @@ class _KQueue(object):
         if events & IOLoop.WRITE:
             kevents.append(select.kevent(
                 fd, filter=select.KQ_FILTER_WRITE, flags=flags))
-        if events & IOLoop.READ or not kevents:
+        if events & IOLoop.READ:
-            # Always read when there is not a write
             kevents.append(select.kevent(
                 fd, filter=select.KQ_FILTER_READ, flags=flags))
         # Even though control() takes a list, it seems to return EINVAL

libs/tornado/platform/select.py

@@ -47,7 +47,7 @@ class _Select(object):
             # Closed connections are reported as errors by epoll and kqueue,
             # but as zero-byte reads by select, so when errors are requested
             # we need to listen for both read and error.
-            self.read_fds.add(fd)
+            #self.read_fds.add(fd)
 
     def modify(self, fd, events):
         self.unregister(fd)

libs/tornado/simple_httpclient.py

@@ -193,12 +193,8 @@ class _HTTPConnection(httputil.HTTPMessageDelegate):
             netloc = self.parsed.netloc
             if "@" in netloc:
                 userpass, _, netloc = netloc.rpartition("@")
-            match = re.match(r'^(.+):(\d+)$', netloc)
+            host, port = httputil.split_host_and_port(netloc)
-            if match:
+            if port is None:
-                host = match.group(1)
-                port = int(match.group(2))
-            else:
-                host = netloc
                 port = 443 if self.parsed.scheme == "https" else 80
             if re.match(r'^\[.*\]$', host):
                 # raw ipv6 addresses in urls are enclosed in brackets

libs/tornado/testing.py

@@ -19,6 +19,7 @@ try:
     from tornado.simple_httpclient import SimpleAsyncHTTPClient
     from tornado.ioloop import IOLoop, TimeoutError
     from tornado import netutil
+    from tornado.process import Subprocess
 except ImportError:
     # These modules are not importable on app engine.  Parts of this module
     # won't work, but e.g. LogTrapTestCase and main() will.
@@ -28,6 +29,7 @@ except ImportError:
     IOLoop = None
     netutil = None
     SimpleAsyncHTTPClient = None
+    Subprocess = None
 from tornado.log import gen_log, app_log
 from tornado.stack_context import ExceptionStackContext
 from tornado.util import raise_exc_info, basestring_type
@@ -214,6 +216,8 @@ class AsyncTestCase(unittest.TestCase):
         self.io_loop.make_current()
 
     def tearDown(self):
+        # Clean up Subprocess, so it can be used again with a new ioloop.
+        Subprocess.uninitialize()
         self.io_loop.clear_current()
         if (not IOLoop.initialized() or
                 self.io_loop is not IOLoop.instance()):

libs/tornado/web.py

@@ -85,6 +85,7 @@ from tornado import stack_context
 from tornado import template
 from tornado.escape import utf8, _unicode
 from tornado.util import import_object, ObjectDict, raise_exc_info, unicode_type, _websocket_mask
+from tornado.httputil import split_host_and_port
 
 
 try:
@@ -1477,7 +1478,7 @@ def asynchronous(method):
         with stack_context.ExceptionStackContext(
                 self._stack_context_handle_exception):
             result = method(self, *args, **kwargs)
-            if isinstance(result, Future):
+            if is_future(result):
                 # If @asynchronous is used with @gen.coroutine, (but
                 # not @gen.engine), we can automatically finish the
                 # request when the future resolves.  Additionally,
@@ -1518,7 +1519,7 @@ def stream_request_body(cls):
       the entire body has been read.
 
     There is a subtle interaction between ``data_received`` and asynchronous
-    ``prepare``: The first call to ``data_recieved`` may occur at any point
+    ``prepare``: The first call to ``data_received`` may occur at any point
     after the call to ``prepare`` has returned *or yielded*.
     """
     if not issubclass(cls, RequestHandler):
@@ -1729,7 +1730,7 @@ class Application(httputil.HTTPServerConnectionDelegate):
         self.transforms.append(transform_class)
 
     def _get_host_handlers(self, request):
-        host = request.host.lower().split(':')[0]
+        host = split_host_and_port(request.host.lower())[0]
         matches = []
         for pattern, handlers in self.handlers:
             if pattern.match(host):
@@ -1845,7 +1846,7 @@ class _RequestDispatcher(httputil.HTTPMessageDelegate):
         handlers = app._get_host_handlers(self.request)
         if not handlers:
             self.handler_class = RedirectHandler
-            self.handler_kwargs = dict(url="http://" + app.default_host + "/")
+            self.handler_kwargs = dict(url="%s://%s/" % (self.request.protocol, app.default_host))
             return
         for spec in handlers:
             match = spec.regex.match(self.request.path)

libs/tornado/websocket.py

@@ -229,7 +229,7 @@ class WebSocketHandler(tornado.web.RequestHandler):
         """
         return None
 
-    def open(self):
+    def open(self, *args, **kwargs):
         """Invoked when a new WebSocket is opened.
 
         The arguments to `open` are extracted from the `tornado.web.URLSpec`

libs/tornado/wsgi.py

@@ -207,7 +207,7 @@ class WSGIAdapter(object):
             body = environ["wsgi.input"].read(
                 int(headers["Content-Length"]))
         else:
-            body = ""
+            body = b""
         protocol = environ["wsgi.url_scheme"]
         remote_ip = environ.get("REMOTE_ADDR", "")
         if environ.get("HTTP_HOST"):