Packages update
This commit is contained in:
@@ -318,7 +318,7 @@ _Translator = {
|
||||
'\375' : '\\375', '\376' : '\\376', '\377' : '\\377'
|
||||
}
|
||||
|
||||
_idmap = ''.join(chr(x) for x in xrange(256))
|
||||
_idmap = ''.join(chr(x) for x in range(256))
|
||||
|
||||
def _quote(str, LegalChars=_LegalChars,
|
||||
idmap=_idmap, translate=string.translate):
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# urllib3/__init__.py
|
||||
# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
|
||||
# Copyright 2008-2012 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
|
||||
#
|
||||
# This module is part of urllib3 and is released under
|
||||
# the MIT License: http://www.opensource.org/licenses/mit-license.php
|
||||
@@ -10,7 +10,7 @@ urllib3 - Thread-safe connection pooling and re-using.
|
||||
|
||||
__author__ = 'Andrey Petrov (andrey.petrov@shazow.net)'
|
||||
__license__ = 'MIT'
|
||||
__version__ = '1.0.2'
|
||||
__version__ = '1.1'
|
||||
|
||||
|
||||
from .connectionpool import (
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# urllib3/_collections.py
|
||||
# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
|
||||
# Copyright 2008-2012 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
|
||||
#
|
||||
# This module is part of urllib3 and is released under
|
||||
# the MIT License: http://www.opensource.org/licenses/mit-license.php
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# urllib3/connectionpool.py
|
||||
# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
|
||||
# Copyright 2008-2012 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
|
||||
#
|
||||
# This module is part of urllib3 and is released under
|
||||
# the MIT License: http://www.opensource.org/licenses/mit-license.php
|
||||
@@ -7,15 +7,27 @@
|
||||
import logging
|
||||
import socket
|
||||
|
||||
|
||||
from httplib import HTTPConnection, HTTPSConnection, HTTPException
|
||||
from Queue import Queue, Empty, Full
|
||||
from select import select
|
||||
from socket import error as SocketError, timeout as SocketTimeout
|
||||
|
||||
from .packages.ssl_match_hostname import match_hostname, CertificateError
|
||||
|
||||
try:
|
||||
from select import poll, POLLIN
|
||||
except ImportError: # Doesn't exist on OSX and other platforms
|
||||
from select import select
|
||||
poll = False
|
||||
|
||||
try: # Python 3
|
||||
from http.client import HTTPConnection, HTTPSConnection, HTTPException
|
||||
from http.client import HTTP_PORT, HTTPS_PORT
|
||||
except ImportError:
|
||||
from httplib import HTTPConnection, HTTPSConnection, HTTPException
|
||||
from httplib import HTTP_PORT, HTTPS_PORT
|
||||
|
||||
try: # Python 3
|
||||
from queue import Queue, Empty, Full
|
||||
except ImportError:
|
||||
from Queue import Queue, Empty, Full
|
||||
|
||||
try: # Compiled with SSL?
|
||||
import ssl
|
||||
BaseSSLError = ssl.SSLError
|
||||
except ImportError:
|
||||
@@ -23,21 +35,29 @@ except ImportError:
|
||||
BaseSSLError = None
|
||||
|
||||
|
||||
from .packages.ssl_match_hostname import match_hostname, CertificateError
|
||||
from .request import RequestMethods
|
||||
from .response import HTTPResponse
|
||||
from .exceptions import (
|
||||
SSLError,
|
||||
from .exceptions import (SSLError,
|
||||
MaxRetryError,
|
||||
TimeoutError,
|
||||
HostChangedError,
|
||||
EmptyPoolError,
|
||||
)
|
||||
|
||||
from .packages.ssl_match_hostname import match_hostname, CertificateError
|
||||
from .packages import six
|
||||
|
||||
xrange = six.moves.xrange
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
_Default = object()
|
||||
|
||||
port_by_scheme = {
|
||||
'http': HTTP_PORT,
|
||||
'https': HTTPS_PORT,
|
||||
}
|
||||
|
||||
## Connection objects (extension of httplib)
|
||||
|
||||
@@ -81,7 +101,16 @@ class ConnectionPool(object):
|
||||
Base class for all connection pools, such as
|
||||
:class:`.HTTPConnectionPool` and :class:`.HTTPSConnectionPool`.
|
||||
"""
|
||||
pass
|
||||
|
||||
scheme = None
|
||||
|
||||
def __init__(self, host, port=None):
|
||||
self.host = host
|
||||
self.port = port
|
||||
|
||||
def __str__(self):
|
||||
return '%s(host=%r, port=%r)' % (type(self).__name__,
|
||||
self.host, self.port)
|
||||
|
||||
|
||||
class HTTPConnectionPool(ConnectionPool, RequestMethods):
|
||||
@@ -169,14 +198,14 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
|
||||
conn = self.pool.get(block=self.block, timeout=timeout)
|
||||
|
||||
# If this is a persistent connection, check if it got disconnected
|
||||
if conn and conn.sock and select([conn.sock], [], [], 0.0)[0]:
|
||||
# Either data is buffered (bad), or the connection is dropped.
|
||||
if conn and conn.sock and is_connection_dropped(conn):
|
||||
log.info("Resetting dropped connection: %s" % self.host)
|
||||
conn.close()
|
||||
|
||||
except Empty:
|
||||
if self.block:
|
||||
raise EmptyPoolError("Pool reached maximum size and no more "
|
||||
raise EmptyPoolError(self,
|
||||
"Pool reached maximum size and no more "
|
||||
"connections are allowed.")
|
||||
pass # Oh well, we'll create a new connection then
|
||||
|
||||
@@ -229,11 +258,17 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
|
||||
def is_same_host(self, url):
|
||||
"""
|
||||
Check if the given ``url`` is a member of the same host as this
|
||||
conncetion pool.
|
||||
connection pool.
|
||||
"""
|
||||
# TODO: Add optional support for socket.gethostbyname checking.
|
||||
scheme, host, port = get_host(url)
|
||||
|
||||
if self.port and not port:
|
||||
# Use explicit default port for comparison when none is given.
|
||||
port = port_by_scheme.get(scheme)
|
||||
|
||||
return (url.startswith('/') or
|
||||
get_host(url) == (self.scheme, self.host, self.port))
|
||||
(scheme, host, port) == (self.scheme, self.host, self.port))
|
||||
|
||||
def urlopen(self, method, url, body=None, headers=None, retries=3,
|
||||
redirect=True, assert_same_host=True, timeout=_Default,
|
||||
@@ -306,7 +341,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
|
||||
headers = self.headers
|
||||
|
||||
if retries < 0:
|
||||
raise MaxRetryError("Max retries exceeded for url: %s" % url)
|
||||
raise MaxRetryError(self, url)
|
||||
|
||||
if timeout is _Default:
|
||||
timeout = self.timeout
|
||||
@@ -320,8 +355,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
|
||||
if self.port:
|
||||
host = "%s:%d" % (host, self.port)
|
||||
|
||||
raise HostChangedError("Connection pool with host '%s' tried to "
|
||||
"open a foreign host: %s" % (host, url))
|
||||
raise HostChangedError(self, url, retries - 1)
|
||||
|
||||
conn = None
|
||||
|
||||
@@ -352,27 +386,29 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
|
||||
# ``response.release_conn()`` is called (implicitly by
|
||||
# ``response.read()``)
|
||||
|
||||
except (Empty), e:
|
||||
except Empty as e:
|
||||
# Timed out by queue
|
||||
raise TimeoutError("Request timed out. (pool_timeout=%s)" %
|
||||
raise TimeoutError(self, "Request timed out. (pool_timeout=%s)" %
|
||||
pool_timeout)
|
||||
|
||||
except (SocketTimeout), e:
|
||||
except SocketTimeout as e:
|
||||
# Timed out by socket
|
||||
raise TimeoutError("Request timed out. (timeout=%s)" %
|
||||
raise TimeoutError(self, "Request timed out. (timeout=%s)" %
|
||||
timeout)
|
||||
|
||||
except (BaseSSLError), e:
|
||||
except BaseSSLError as e:
|
||||
# SSL certificate error
|
||||
raise SSLError(e)
|
||||
|
||||
except (CertificateError), e:
|
||||
except CertificateError as e:
|
||||
# Name mismatch
|
||||
raise SSLError(e)
|
||||
|
||||
except (HTTPException, SocketError), e:
|
||||
|
||||
except (HTTPException, SocketError) as e:
|
||||
# Connection broken, discard. It will be replaced next _get_conn().
|
||||
conn = None
|
||||
# This is necessary so we can access e below
|
||||
err = e
|
||||
|
||||
finally:
|
||||
if conn and release_conn:
|
||||
@@ -381,19 +417,16 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
|
||||
|
||||
if not conn:
|
||||
log.warn("Retrying (%d attempts remain) after connection "
|
||||
"broken by '%r': %s" % (retries, e, url))
|
||||
"broken by '%r': %s" % (retries, err, url))
|
||||
return self.urlopen(method, url, body, headers, retries - 1,
|
||||
redirect, assert_same_host) # Try again
|
||||
|
||||
# Handle redirection
|
||||
if (redirect and
|
||||
response.status in [301, 302, 303, 307] and
|
||||
'location' in response.headers): # Redirect, retry
|
||||
log.info("Redirecting %s -> %s" %
|
||||
(url, response.headers.get('location')))
|
||||
return self.urlopen(method, response.headers.get('location'), body,
|
||||
headers, retries - 1, redirect,
|
||||
assert_same_host)
|
||||
# Handle redirect?
|
||||
redirect_location = redirect and response.get_redirect_location()
|
||||
if redirect_location:
|
||||
log.info("Redirecting %s -> %s" % (url, redirect_location))
|
||||
return self.urlopen(method, redirect_location, body, headers,
|
||||
retries - 1, redirect, assert_same_host)
|
||||
|
||||
return response
|
||||
|
||||
@@ -550,3 +583,22 @@ def connection_from_url(url, **kw):
|
||||
return HTTPSConnectionPool(host, port=port, **kw)
|
||||
else:
|
||||
return HTTPConnectionPool(host, port=port, **kw)
|
||||
|
||||
|
||||
def is_connection_dropped(conn):
|
||||
"""
|
||||
Returns True if the connection is dropped and should be closed.
|
||||
|
||||
:param conn:
|
||||
``HTTPConnection`` object.
|
||||
"""
|
||||
if not poll:
|
||||
return select([conn.sock], [], [], 0.0)[0]
|
||||
|
||||
# This version is better on platforms that support it.
|
||||
p = poll()
|
||||
p.register(conn.sock, POLLIN)
|
||||
for (fno, ev) in p.poll(0.0):
|
||||
if fno == conn.sock.fileno():
|
||||
# Either data is buffered (bad), or the connection is dropped.
|
||||
return True
|
||||
|
||||
@@ -1,35 +1,54 @@
|
||||
# urllib3/exceptions.py
|
||||
# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
|
||||
# Copyright 2008-2012 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
|
||||
#
|
||||
# This module is part of urllib3 and is released under
|
||||
# the MIT License: http://www.opensource.org/licenses/mit-license.php
|
||||
|
||||
## Exceptions
|
||||
## Base Exceptions
|
||||
|
||||
class HTTPError(Exception):
|
||||
"Base exception used by this module."
|
||||
pass
|
||||
|
||||
|
||||
class SSLError(Exception):
|
||||
class PoolError(HTTPError):
|
||||
"Base exception for errors caused within a pool."
|
||||
def __init__(self, pool, message):
|
||||
self.pool = pool
|
||||
HTTPError.__init__(self, "%s: %s" % (pool, message))
|
||||
|
||||
|
||||
class SSLError(HTTPError):
|
||||
"Raised when SSL certificate fails in an HTTPS connection."
|
||||
pass
|
||||
|
||||
|
||||
class MaxRetryError(HTTPError):
|
||||
## Leaf Exceptions
|
||||
|
||||
class MaxRetryError(PoolError):
|
||||
"Raised when the maximum number of retries is exceeded."
|
||||
pass
|
||||
def __init__(self, pool, url):
|
||||
PoolError.__init__(self, pool,
|
||||
"Max retries exceeded with url: %s" % url)
|
||||
|
||||
self.url = url
|
||||
|
||||
|
||||
class TimeoutError(HTTPError):
|
||||
class HostChangedError(PoolError):
|
||||
"Raised when an existing pool gets a request for a foreign host."
|
||||
def __init__(self, pool, url, retries=3):
|
||||
PoolError.__init__(self, pool,
|
||||
"Tried to open a foreign host with url: %s" % url)
|
||||
|
||||
self.url = url
|
||||
self.retries = retries
|
||||
|
||||
|
||||
class TimeoutError(PoolError):
|
||||
"Raised when a socket timeout occurs."
|
||||
pass
|
||||
|
||||
|
||||
class HostChangedError(HTTPError):
|
||||
"Raised when an existing pool gets a request for a foreign host."
|
||||
pass
|
||||
|
||||
class EmptyPoolError(HTTPError):
|
||||
class EmptyPoolError(PoolError):
|
||||
"Raised when a pool runs out of connections and no more are allowed."
|
||||
pass
|
||||
|
||||
@@ -1,18 +1,21 @@
|
||||
# urllib3/filepost.py
|
||||
# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
|
||||
# Copyright 2008-2012 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
|
||||
#
|
||||
# This module is part of urllib3 and is released under
|
||||
# the MIT License: http://www.opensource.org/licenses/mit-license.php
|
||||
|
||||
import codecs
|
||||
import mimetools
|
||||
import mimetypes
|
||||
|
||||
try:
|
||||
from cStringIO import StringIO
|
||||
from mimetools import choose_boundary
|
||||
except ImportError:
|
||||
from StringIO import StringIO # pylint: disable-msg=W0404
|
||||
from .packages.mimetools_choose_boundary import choose_boundary
|
||||
|
||||
from io import BytesIO
|
||||
|
||||
from .packages import six
|
||||
from .packages.six import b
|
||||
|
||||
writer = codecs.lookup('utf-8')[3]
|
||||
|
||||
@@ -35,37 +38,37 @@ def encode_multipart_formdata(fields, boundary=None):
|
||||
If not specified, then a random boundary will be generated using
|
||||
:func:`mimetools.choose_boundary`.
|
||||
"""
|
||||
body = StringIO()
|
||||
body = BytesIO()
|
||||
if boundary is None:
|
||||
boundary = mimetools.choose_boundary()
|
||||
boundary = choose_boundary()
|
||||
|
||||
for fieldname, value in fields.iteritems():
|
||||
body.write('--%s\r\n' % (boundary))
|
||||
for fieldname, value in six.iteritems(fields):
|
||||
body.write(b('--%s\r\n' % (boundary)))
|
||||
|
||||
if isinstance(value, tuple):
|
||||
filename, data = value
|
||||
writer(body).write('Content-Disposition: form-data; name="%s"; '
|
||||
'filename="%s"\r\n' % (fieldname, filename))
|
||||
body.write('Content-Type: %s\r\n\r\n' %
|
||||
(get_content_type(filename)))
|
||||
body.write(b('Content-Type: %s\r\n\r\n' %
|
||||
(get_content_type(filename))))
|
||||
else:
|
||||
data = value
|
||||
writer(body).write('Content-Disposition: form-data; name="%s"\r\n'
|
||||
% (fieldname))
|
||||
body.write('Content-Type: text/plain\r\n\r\n')
|
||||
body.write(b'Content-Type: text/plain\r\n\r\n')
|
||||
|
||||
if isinstance(data, int):
|
||||
data = str(data) # Backwards compatibility
|
||||
|
||||
if isinstance(data, unicode):
|
||||
if isinstance(data, six.text_type):
|
||||
writer(body).write(data)
|
||||
else:
|
||||
body.write(data)
|
||||
|
||||
body.write('\r\n')
|
||||
body.write(b'\r\n')
|
||||
|
||||
body.write('--%s--\r\n' % (boundary))
|
||||
body.write(b('--%s--\r\n' % (boundary)))
|
||||
|
||||
content_type = 'multipart/form-data; boundary=%s' % boundary
|
||||
content_type = b('multipart/form-data; boundary=%s' % boundary)
|
||||
|
||||
return body.getvalue(), content_type
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
"""The function mimetools.choose_boundary() from Python 2.7, which seems to
|
||||
have disappeared in Python 3 (although email.generator._make_boundary() might
|
||||
work as a replacement?).
|
||||
|
||||
Tweaked to use lock from threading rather than thread.
|
||||
"""
|
||||
import os
|
||||
from threading import Lock
|
||||
_counter_lock = Lock()
|
||||
|
||||
_counter = 0
|
||||
def _get_next_counter():
|
||||
global _counter
|
||||
with _counter_lock:
|
||||
_counter += 1
|
||||
return _counter
|
||||
|
||||
_prefix = None
|
||||
|
||||
def choose_boundary():
|
||||
"""Return a string usable as a multipart boundary.
|
||||
|
||||
The string chosen is unique within a single program run, and
|
||||
incorporates the user id (if available), process id (if available),
|
||||
and current time. So it's very unlikely the returned string appears
|
||||
in message text, but there's no guarantee.
|
||||
|
||||
The boundary contains dots so you have to quote it in the header."""
|
||||
|
||||
global _prefix
|
||||
import time
|
||||
if _prefix is None:
|
||||
import socket
|
||||
try:
|
||||
hostid = socket.gethostbyname(socket.gethostname())
|
||||
except socket.gaierror:
|
||||
hostid = '127.0.0.1'
|
||||
try:
|
||||
uid = repr(os.getuid())
|
||||
except AttributeError:
|
||||
uid = '1'
|
||||
try:
|
||||
pid = repr(os.getpid())
|
||||
except AttributeError:
|
||||
pid = '1'
|
||||
_prefix = hostid + '.' + uid + '.' + pid
|
||||
return "%s.%.3f.%d" % (_prefix, time.time(), _get_next_counter())
|
||||
@@ -0,0 +1,372 @@
|
||||
"""Utilities for writing code that runs on Python 2 and 3"""
|
||||
|
||||
#Copyright (c) 2010-2011 Benjamin Peterson
|
||||
|
||||
#Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||
#this software and associated documentation files (the "Software"), to deal in
|
||||
#the Software without restriction, including without limitation the rights to
|
||||
#use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
|
||||
#the Software, and to permit persons to whom the Software is furnished to do so,
|
||||
#subject to the following conditions:
|
||||
|
||||
#The above copyright notice and this permission notice shall be included in all
|
||||
#copies or substantial portions of the Software.
|
||||
|
||||
#THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
#IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
||||
#FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
||||
#COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
|
||||
#IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
|
||||
#CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
|
||||
import operator
|
||||
import sys
|
||||
import types
|
||||
|
||||
__author__ = "Benjamin Peterson <benjamin@python.org>"
|
||||
__version__ = "1.1.0"
|
||||
|
||||
|
||||
# True if we are running on Python 3.
|
||||
PY3 = sys.version_info[0] == 3
|
||||
|
||||
if PY3:
|
||||
string_types = str,
|
||||
integer_types = int,
|
||||
class_types = type,
|
||||
text_type = str
|
||||
binary_type = bytes
|
||||
|
||||
MAXSIZE = sys.maxsize
|
||||
else:
|
||||
string_types = basestring,
|
||||
integer_types = (int, long)
|
||||
class_types = (type, types.ClassType)
|
||||
text_type = unicode
|
||||
binary_type = str
|
||||
|
||||
# It's possible to have sizeof(long) != sizeof(Py_ssize_t).
|
||||
class X(object):
|
||||
def __len__(self):
|
||||
return 1 << 31
|
||||
try:
|
||||
len(X())
|
||||
except OverflowError:
|
||||
# 32-bit
|
||||
MAXSIZE = int((1 << 31) - 1)
|
||||
else:
|
||||
# 64-bit
|
||||
MAXSIZE = int((1 << 63) - 1)
|
||||
del X
|
||||
|
||||
|
||||
def _add_doc(func, doc):
|
||||
"""Add documentation to a function."""
|
||||
func.__doc__ = doc
|
||||
|
||||
|
||||
def _import_module(name):
|
||||
"""Import module, returning the module after the last dot."""
|
||||
__import__(name)
|
||||
return sys.modules[name]
|
||||
|
||||
|
||||
class _LazyDescr(object):
|
||||
|
||||
def __init__(self, name):
|
||||
self.name = name
|
||||
|
||||
def __get__(self, obj, tp):
|
||||
result = self._resolve()
|
||||
setattr(obj, self.name, result)
|
||||
# This is a bit ugly, but it avoids running this again.
|
||||
delattr(tp, self.name)
|
||||
return result
|
||||
|
||||
|
||||
class MovedModule(_LazyDescr):
|
||||
|
||||
def __init__(self, name, old, new=None):
|
||||
super(MovedModule, self).__init__(name)
|
||||
if PY3:
|
||||
if new is None:
|
||||
new = name
|
||||
self.mod = new
|
||||
else:
|
||||
self.mod = old
|
||||
|
||||
def _resolve(self):
|
||||
return _import_module(self.mod)
|
||||
|
||||
|
||||
class MovedAttribute(_LazyDescr):
|
||||
|
||||
def __init__(self, name, old_mod, new_mod, old_attr=None, new_attr=None):
|
||||
super(MovedAttribute, self).__init__(name)
|
||||
if PY3:
|
||||
if new_mod is None:
|
||||
new_mod = name
|
||||
self.mod = new_mod
|
||||
if new_attr is None:
|
||||
if old_attr is None:
|
||||
new_attr = name
|
||||
else:
|
||||
new_attr = old_attr
|
||||
self.attr = new_attr
|
||||
else:
|
||||
self.mod = old_mod
|
||||
if old_attr is None:
|
||||
old_attr = name
|
||||
self.attr = old_attr
|
||||
|
||||
def _resolve(self):
|
||||
module = _import_module(self.mod)
|
||||
return getattr(module, self.attr)
|
||||
|
||||
|
||||
|
||||
class _MovedItems(types.ModuleType):
|
||||
"""Lazy loading of moved objects"""
|
||||
|
||||
|
||||
_moved_attributes = [
|
||||
MovedAttribute("cStringIO", "cStringIO", "io", "StringIO"),
|
||||
MovedAttribute("filter", "itertools", "builtins", "ifilter", "filter"),
|
||||
MovedAttribute("map", "itertools", "builtins", "imap", "map"),
|
||||
MovedAttribute("reload_module", "__builtin__", "imp", "reload"),
|
||||
MovedAttribute("reduce", "__builtin__", "functools"),
|
||||
MovedAttribute("StringIO", "StringIO", "io"),
|
||||
MovedAttribute("xrange", "__builtin__", "builtins", "xrange", "range"),
|
||||
MovedAttribute("zip", "itertools", "builtins", "izip", "zip"),
|
||||
|
||||
MovedModule("builtins", "__builtin__"),
|
||||
MovedModule("configparser", "ConfigParser"),
|
||||
MovedModule("copyreg", "copy_reg"),
|
||||
MovedModule("http_cookiejar", "cookielib", "http.cookiejar"),
|
||||
MovedModule("http_cookies", "Cookie", "http.cookies"),
|
||||
MovedModule("html_entities", "htmlentitydefs", "html.entities"),
|
||||
MovedModule("html_parser", "HTMLParser", "html.parser"),
|
||||
MovedModule("http_client", "httplib", "http.client"),
|
||||
MovedModule("BaseHTTPServer", "BaseHTTPServer", "http.server"),
|
||||
MovedModule("CGIHTTPServer", "CGIHTTPServer", "http.server"),
|
||||
MovedModule("SimpleHTTPServer", "SimpleHTTPServer", "http.server"),
|
||||
MovedModule("cPickle", "cPickle", "pickle"),
|
||||
MovedModule("queue", "Queue"),
|
||||
MovedModule("reprlib", "repr"),
|
||||
MovedModule("socketserver", "SocketServer"),
|
||||
MovedModule("tkinter", "Tkinter"),
|
||||
MovedModule("tkinter_dialog", "Dialog", "tkinter.dialog"),
|
||||
MovedModule("tkinter_filedialog", "FileDialog", "tkinter.filedialog"),
|
||||
MovedModule("tkinter_scrolledtext", "ScrolledText", "tkinter.scrolledtext"),
|
||||
MovedModule("tkinter_simpledialog", "SimpleDialog", "tkinter.simpledialog"),
|
||||
MovedModule("tkinter_tix", "Tix", "tkinter.tix"),
|
||||
MovedModule("tkinter_constants", "Tkconstants", "tkinter.constants"),
|
||||
MovedModule("tkinter_dnd", "Tkdnd", "tkinter.dnd"),
|
||||
MovedModule("tkinter_colorchooser", "tkColorChooser",
|
||||
"tkinter.colorchooser"),
|
||||
MovedModule("tkinter_commondialog", "tkCommonDialog",
|
||||
"tkinter.commondialog"),
|
||||
MovedModule("tkinter_tkfiledialog", "tkFileDialog", "tkinter.filedialog"),
|
||||
MovedModule("tkinter_font", "tkFont", "tkinter.font"),
|
||||
MovedModule("tkinter_messagebox", "tkMessageBox", "tkinter.messagebox"),
|
||||
MovedModule("tkinter_tksimpledialog", "tkSimpleDialog",
|
||||
"tkinter.simpledialog"),
|
||||
MovedModule("urllib_robotparser", "robotparser", "urllib.robotparser"),
|
||||
MovedModule("winreg", "_winreg"),
|
||||
]
|
||||
for attr in _moved_attributes:
|
||||
setattr(_MovedItems, attr.name, attr)
|
||||
del attr
|
||||
|
||||
moves = sys.modules["six.moves"] = _MovedItems("moves")
|
||||
|
||||
|
||||
def add_move(move):
|
||||
"""Add an item to six.moves."""
|
||||
setattr(_MovedItems, move.name, move)
|
||||
|
||||
|
||||
def remove_move(name):
|
||||
"""Remove item from six.moves."""
|
||||
try:
|
||||
delattr(_MovedItems, name)
|
||||
except AttributeError:
|
||||
try:
|
||||
del moves.__dict__[name]
|
||||
except KeyError:
|
||||
raise AttributeError("no such move, %r" % (name,))
|
||||
|
||||
|
||||
if PY3:
|
||||
_meth_func = "__func__"
|
||||
_meth_self = "__self__"
|
||||
|
||||
_func_code = "__code__"
|
||||
_func_defaults = "__defaults__"
|
||||
|
||||
_iterkeys = "keys"
|
||||
_itervalues = "values"
|
||||
_iteritems = "items"
|
||||
else:
|
||||
_meth_func = "im_func"
|
||||
_meth_self = "im_self"
|
||||
|
||||
_func_code = "func_code"
|
||||
_func_defaults = "func_defaults"
|
||||
|
||||
_iterkeys = "iterkeys"
|
||||
_itervalues = "itervalues"
|
||||
_iteritems = "iteritems"
|
||||
|
||||
|
||||
if PY3:
|
||||
def get_unbound_function(unbound):
|
||||
return unbound
|
||||
|
||||
|
||||
advance_iterator = next
|
||||
|
||||
def callable(obj):
|
||||
return any("__call__" in klass.__dict__ for klass in type(obj).__mro__)
|
||||
else:
|
||||
def get_unbound_function(unbound):
|
||||
return unbound.im_func
|
||||
|
||||
|
||||
def advance_iterator(it):
|
||||
return it.next()
|
||||
|
||||
callable = callable
|
||||
_add_doc(get_unbound_function,
|
||||
"""Get the function out of a possibly unbound function""")
|
||||
|
||||
|
||||
get_method_function = operator.attrgetter(_meth_func)
|
||||
get_method_self = operator.attrgetter(_meth_self)
|
||||
get_function_code = operator.attrgetter(_func_code)
|
||||
get_function_defaults = operator.attrgetter(_func_defaults)
|
||||
|
||||
|
||||
def iterkeys(d):
|
||||
"""Return an iterator over the keys of a dictionary."""
|
||||
return getattr(d, _iterkeys)()
|
||||
|
||||
def itervalues(d):
|
||||
"""Return an iterator over the values of a dictionary."""
|
||||
return getattr(d, _itervalues)()
|
||||
|
||||
def iteritems(d):
|
||||
"""Return an iterator over the (key, value) pairs of a dictionary."""
|
||||
return getattr(d, _iteritems)()
|
||||
|
||||
|
||||
if PY3:
|
||||
def b(s):
|
||||
return s.encode("latin-1")
|
||||
def u(s):
|
||||
return s
|
||||
if sys.version_info[1] <= 1:
|
||||
def int2byte(i):
|
||||
return bytes((i,))
|
||||
else:
|
||||
# This is about 2x faster than the implementation above on 3.2+
|
||||
int2byte = operator.methodcaller("to_bytes", 1, "big")
|
||||
import io
|
||||
StringIO = io.StringIO
|
||||
BytesIO = io.BytesIO
|
||||
else:
|
||||
def b(s):
|
||||
return s
|
||||
def u(s):
|
||||
return unicode(s, "unicode_escape")
|
||||
int2byte = chr
|
||||
import StringIO
|
||||
StringIO = BytesIO = StringIO.StringIO
|
||||
_add_doc(b, """Byte literal""")
|
||||
_add_doc(u, """Text literal""")
|
||||
|
||||
|
||||
if PY3:
|
||||
import builtins
|
||||
exec_ = getattr(builtins, "exec")
|
||||
|
||||
|
||||
def reraise(tp, value, tb=None):
|
||||
if value.__traceback__ is not tb:
|
||||
raise value.with_traceback(tb)
|
||||
raise value
|
||||
|
||||
|
||||
print_ = getattr(builtins, "print")
|
||||
del builtins
|
||||
|
||||
else:
|
||||
def exec_(code, globs=None, locs=None):
|
||||
"""Execute code in a namespace."""
|
||||
if globs is None:
|
||||
frame = sys._getframe(1)
|
||||
globs = frame.f_globals
|
||||
if locs is None:
|
||||
locs = frame.f_locals
|
||||
del frame
|
||||
elif locs is None:
|
||||
locs = globs
|
||||
exec("""exec code in globs, locs""")
|
||||
|
||||
|
||||
exec_("""def reraise(tp, value, tb=None):
|
||||
raise tp, value, tb
|
||||
""")
|
||||
|
||||
|
||||
def print_(*args, **kwargs):
|
||||
"""The new-style print function."""
|
||||
fp = kwargs.pop("file", sys.stdout)
|
||||
if fp is None:
|
||||
return
|
||||
def write(data):
|
||||
if not isinstance(data, basestring):
|
||||
data = str(data)
|
||||
fp.write(data)
|
||||
want_unicode = False
|
||||
sep = kwargs.pop("sep", None)
|
||||
if sep is not None:
|
||||
if isinstance(sep, unicode):
|
||||
want_unicode = True
|
||||
elif not isinstance(sep, str):
|
||||
raise TypeError("sep must be None or a string")
|
||||
end = kwargs.pop("end", None)
|
||||
if end is not None:
|
||||
if isinstance(end, unicode):
|
||||
want_unicode = True
|
||||
elif not isinstance(end, str):
|
||||
raise TypeError("end must be None or a string")
|
||||
if kwargs:
|
||||
raise TypeError("invalid keyword arguments to print()")
|
||||
if not want_unicode:
|
||||
for arg in args:
|
||||
if isinstance(arg, unicode):
|
||||
want_unicode = True
|
||||
break
|
||||
if want_unicode:
|
||||
newline = unicode("\n")
|
||||
space = unicode(" ")
|
||||
else:
|
||||
newline = "\n"
|
||||
space = " "
|
||||
if sep is None:
|
||||
sep = space
|
||||
if end is None:
|
||||
end = newline
|
||||
for i, arg in enumerate(args):
|
||||
if i:
|
||||
write(sep)
|
||||
write(arg)
|
||||
write(end)
|
||||
|
||||
_add_doc(reraise, """Reraise an exception.""")
|
||||
|
||||
|
||||
def with_metaclass(meta, base=object):
|
||||
"""Create a base class with a metaclass."""
|
||||
return meta("NewBase", (base,), {})
|
||||
@@ -1,32 +1,27 @@
|
||||
# urllib3/poolmanager.py
|
||||
# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
|
||||
# Copyright 2008-2012 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
|
||||
#
|
||||
# This module is part of urllib3 and is released under
|
||||
# the MIT License: http://www.opensource.org/licenses/mit-license.php
|
||||
|
||||
import logging
|
||||
|
||||
from ._collections import RecentlyUsedContainer
|
||||
from .connectionpool import (
|
||||
HTTPConnectionPool, HTTPSConnectionPool,
|
||||
get_host, connection_from_url,
|
||||
)
|
||||
from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool
|
||||
from .connectionpool import get_host, connection_from_url, port_by_scheme
|
||||
from .exceptions import HostChangedError
|
||||
from .request import RequestMethods
|
||||
|
||||
|
||||
__all__ = ['PoolManager', 'ProxyManager', 'proxy_from_url']
|
||||
|
||||
|
||||
from .request import RequestMethods
|
||||
from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool
|
||||
|
||||
|
||||
pool_classes_by_scheme = {
|
||||
'http': HTTPConnectionPool,
|
||||
'https': HTTPSConnectionPool,
|
||||
}
|
||||
|
||||
port_by_scheme = {
|
||||
'http': 80,
|
||||
'https': 443,
|
||||
}
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class PoolManager(RequestMethods):
|
||||
@@ -105,7 +100,12 @@ class PoolManager(RequestMethods):
|
||||
:class:`urllib3.connectionpool.ConnectionPool` can be chosen for it.
|
||||
"""
|
||||
conn = self.connection_from_url(url)
|
||||
return conn.urlopen(method, url, assert_same_host=False, **kw)
|
||||
try:
|
||||
return conn.urlopen(method, url, **kw)
|
||||
|
||||
except HostChangedError as e:
|
||||
kw['retries'] = e.retries # Persist retries countdown
|
||||
return self.urlopen(method, e.url, **kw)
|
||||
|
||||
|
||||
class ProxyManager(RequestMethods):
|
||||
|
||||
@@ -1,11 +1,13 @@
|
||||
# urllib3/request.py
|
||||
# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
|
||||
# Copyright 2008-2012 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
|
||||
#
|
||||
# This module is part of urllib3 and is released under
|
||||
# the MIT License: http://www.opensource.org/licenses/mit-license.php
|
||||
|
||||
|
||||
from urllib import urlencode
|
||||
try:
|
||||
from urllib.parse import urlencode
|
||||
except ImportError:
|
||||
from urllib import urlencode
|
||||
|
||||
from .filepost import encode_multipart_formdata
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# urllib3/response.py
|
||||
# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
|
||||
# Copyright 2008-2012 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
|
||||
#
|
||||
# This module is part of urllib3 and is released under
|
||||
# the MIT License: http://www.opensource.org/licenses/mit-license.php
|
||||
@@ -8,21 +8,22 @@ import gzip
|
||||
import logging
|
||||
import zlib
|
||||
|
||||
|
||||
try:
|
||||
from cStringIO import StringIO
|
||||
except ImportError:
|
||||
from StringIO import StringIO # pylint: disable-msg=W0404
|
||||
|
||||
from io import BytesIO
|
||||
|
||||
from .exceptions import HTTPError
|
||||
|
||||
|
||||
try:
|
||||
basestring = basestring
|
||||
except NameError: # Python 3
|
||||
basestring = (str, bytes)
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def decode_gzip(data):
|
||||
gzipper = gzip.GzipFile(fileobj=StringIO(data))
|
||||
gzipper = gzip.GzipFile(fileobj=BytesIO(data))
|
||||
return gzipper.read()
|
||||
|
||||
|
||||
@@ -71,7 +72,7 @@ class HTTPResponse(object):
|
||||
self.strict = strict
|
||||
|
||||
self._decode_content = decode_content
|
||||
self._body = None
|
||||
self._body = body if body and isinstance(body, basestring) else None
|
||||
self._fp = None
|
||||
self._original_response = original_response
|
||||
|
||||
@@ -81,9 +82,22 @@ class HTTPResponse(object):
|
||||
if hasattr(body, 'read'):
|
||||
self._fp = body
|
||||
|
||||
if preload_content:
|
||||
if preload_content and not self._body:
|
||||
self._body = self.read(decode_content=decode_content)
|
||||
|
||||
def get_redirect_location(self):
|
||||
"""
|
||||
Should we redirect and where to?
|
||||
|
||||
:returns: Truthy redirect location string if we got a redirect status
|
||||
code and valid location. ``None`` if redirect status and no
|
||||
location. ``False`` if not a redirect status code.
|
||||
"""
|
||||
if self.status in [301, 302, 303, 307]:
|
||||
return self.headers.get('location')
|
||||
|
||||
return False
|
||||
|
||||
def release_conn(self):
|
||||
if not self._pool or not self._connection:
|
||||
return
|
||||
@@ -98,10 +112,9 @@ class HTTPResponse(object):
|
||||
return self._body
|
||||
|
||||
if self._fp:
|
||||
return self.read(decode_content=self._decode_content,
|
||||
cache_content=True)
|
||||
return self.read(cache_content=True)
|
||||
|
||||
def read(self, amt=None, decode_content=True, cache_content=False):
|
||||
def read(self, amt=None, decode_content=None, cache_content=False):
|
||||
"""
|
||||
Similar to :meth:`httplib.HTTPResponse.read`, but with two additional
|
||||
parameters: ``decode_content`` and ``cache_content``.
|
||||
@@ -124,22 +137,22 @@ class HTTPResponse(object):
|
||||
"""
|
||||
content_encoding = self.headers.get('content-encoding')
|
||||
decoder = self.CONTENT_DECODERS.get(content_encoding)
|
||||
if decode_content is None:
|
||||
decode_content = self._decode_content
|
||||
|
||||
data = self._fp and self._fp.read(amt)
|
||||
if self._fp is None:
|
||||
return
|
||||
|
||||
try:
|
||||
|
||||
if amt:
|
||||
return data
|
||||
|
||||
if not decode_content or not decoder:
|
||||
if cache_content:
|
||||
self._body = data
|
||||
|
||||
return data
|
||||
if amt is None:
|
||||
# cStringIO doesn't like amt=None
|
||||
data = self._fp.read()
|
||||
else:
|
||||
return self._fp.read(amt)
|
||||
|
||||
try:
|
||||
data = decoder(data)
|
||||
if decode_content and decoder:
|
||||
data = decoder(data)
|
||||
except IOError:
|
||||
raise HTTPError("Received response with content-encoding: %s, but "
|
||||
"failed to decode it." % content_encoding)
|
||||
@@ -150,12 +163,11 @@ class HTTPResponse(object):
|
||||
return data
|
||||
|
||||
finally:
|
||||
|
||||
if self._original_response and self._original_response.isclosed():
|
||||
self.release_conn()
|
||||
|
||||
@staticmethod
|
||||
def from_httplib(r, **response_kw):
|
||||
@classmethod
|
||||
def from_httplib(ResponseCls, r, **response_kw):
|
||||
"""
|
||||
Given an :class:`httplib.HTTPResponse` instance ``r``, return a
|
||||
corresponding :class:`urllib3.response.HTTPResponse` object.
|
||||
@@ -164,14 +176,17 @@ class HTTPResponse(object):
|
||||
with ``original_response=r``.
|
||||
"""
|
||||
|
||||
return HTTPResponse(body=r,
|
||||
headers=dict(r.getheaders()),
|
||||
status=r.status,
|
||||
version=r.version,
|
||||
reason=r.reason,
|
||||
strict=r.strict,
|
||||
original_response=r,
|
||||
**response_kw)
|
||||
# HTTPResponse objects in Python 3 don't have a .strict attribute
|
||||
strict = getattr(r, 'strict', 0)
|
||||
return ResponseCls(body=r,
|
||||
# In Python 3, the header keys are returned capitalised
|
||||
headers=dict((k.lower(), v) for k,v in r.getheaders()),
|
||||
status=r.status,
|
||||
version=r.version,
|
||||
reason=r.reason,
|
||||
strict=strict,
|
||||
original_response=r,
|
||||
**response_kw)
|
||||
|
||||
# Backwards-compatibility methods for httplib.HTTPResponse
|
||||
def getheaders(self):
|
||||
|
||||
Reference in New Issue
Block a user