Packages update

This commit is contained in:
Ruud
2012-02-11 16:28:06 +01:00
parent 3bbf1126c3
commit 02e01fb2d6
217 changed files with 26395 additions and 21194 deletions
+1 -1
View File
@@ -318,7 +318,7 @@ _Translator = {
'\375' : '\\375', '\376' : '\\376', '\377' : '\\377'
}
_idmap = ''.join(chr(x) for x in xrange(256))
_idmap = ''.join(chr(x) for x in range(256))
def _quote(str, LegalChars=_LegalChars,
idmap=_idmap, translate=string.translate):
+2 -2
View File
@@ -1,5 +1,5 @@
# urllib3/__init__.py
# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
# Copyright 2008-2012 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
#
# This module is part of urllib3 and is released under
# the MIT License: http://www.opensource.org/licenses/mit-license.php
@@ -10,7 +10,7 @@ urllib3 - Thread-safe connection pooling and re-using.
__author__ = 'Andrey Petrov (andrey.petrov@shazow.net)'
__license__ = 'MIT'
__version__ = '1.0.2'
__version__ = '1.1'
from .connectionpool import (
@@ -1,5 +1,5 @@
# urllib3/_collections.py
# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
# Copyright 2008-2012 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
#
# This module is part of urllib3 and is released under
# the MIT License: http://www.opensource.org/licenses/mit-license.php
@@ -1,5 +1,5 @@
# urllib3/connectionpool.py
# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
# Copyright 2008-2012 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
#
# This module is part of urllib3 and is released under
# the MIT License: http://www.opensource.org/licenses/mit-license.php
@@ -7,15 +7,27 @@
import logging
import socket
from httplib import HTTPConnection, HTTPSConnection, HTTPException
from Queue import Queue, Empty, Full
from select import select
from socket import error as SocketError, timeout as SocketTimeout
from .packages.ssl_match_hostname import match_hostname, CertificateError
try:
from select import poll, POLLIN
except ImportError: # Doesn't exist on OSX and other platforms
from select import select
poll = False
try: # Python 3
from http.client import HTTPConnection, HTTPSConnection, HTTPException
from http.client import HTTP_PORT, HTTPS_PORT
except ImportError:
from httplib import HTTPConnection, HTTPSConnection, HTTPException
from httplib import HTTP_PORT, HTTPS_PORT
try: # Python 3
from queue import Queue, Empty, Full
except ImportError:
from Queue import Queue, Empty, Full
try: # Compiled with SSL?
import ssl
BaseSSLError = ssl.SSLError
except ImportError:
@@ -23,21 +35,29 @@ except ImportError:
BaseSSLError = None
from .packages.ssl_match_hostname import match_hostname, CertificateError
from .request import RequestMethods
from .response import HTTPResponse
from .exceptions import (
SSLError,
from .exceptions import (SSLError,
MaxRetryError,
TimeoutError,
HostChangedError,
EmptyPoolError,
)
from .packages.ssl_match_hostname import match_hostname, CertificateError
from .packages import six
xrange = six.moves.xrange
log = logging.getLogger(__name__)
_Default = object()
port_by_scheme = {
'http': HTTP_PORT,
'https': HTTPS_PORT,
}
## Connection objects (extension of httplib)
@@ -81,7 +101,16 @@ class ConnectionPool(object):
Base class for all connection pools, such as
:class:`.HTTPConnectionPool` and :class:`.HTTPSConnectionPool`.
"""
pass
scheme = None
def __init__(self, host, port=None):
self.host = host
self.port = port
def __str__(self):
return '%s(host=%r, port=%r)' % (type(self).__name__,
self.host, self.port)
class HTTPConnectionPool(ConnectionPool, RequestMethods):
@@ -169,14 +198,14 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
conn = self.pool.get(block=self.block, timeout=timeout)
# If this is a persistent connection, check if it got disconnected
if conn and conn.sock and select([conn.sock], [], [], 0.0)[0]:
# Either data is buffered (bad), or the connection is dropped.
if conn and conn.sock and is_connection_dropped(conn):
log.info("Resetting dropped connection: %s" % self.host)
conn.close()
except Empty:
if self.block:
raise EmptyPoolError("Pool reached maximum size and no more "
raise EmptyPoolError(self,
"Pool reached maximum size and no more "
"connections are allowed.")
pass # Oh well, we'll create a new connection then
@@ -229,11 +258,17 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
def is_same_host(self, url):
"""
Check if the given ``url`` is a member of the same host as this
conncetion pool.
connection pool.
"""
# TODO: Add optional support for socket.gethostbyname checking.
scheme, host, port = get_host(url)
if self.port and not port:
# Use explicit default port for comparison when none is given.
port = port_by_scheme.get(scheme)
return (url.startswith('/') or
get_host(url) == (self.scheme, self.host, self.port))
(scheme, host, port) == (self.scheme, self.host, self.port))
def urlopen(self, method, url, body=None, headers=None, retries=3,
redirect=True, assert_same_host=True, timeout=_Default,
@@ -306,7 +341,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
headers = self.headers
if retries < 0:
raise MaxRetryError("Max retries exceeded for url: %s" % url)
raise MaxRetryError(self, url)
if timeout is _Default:
timeout = self.timeout
@@ -320,8 +355,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
if self.port:
host = "%s:%d" % (host, self.port)
raise HostChangedError("Connection pool with host '%s' tried to "
"open a foreign host: %s" % (host, url))
raise HostChangedError(self, url, retries - 1)
conn = None
@@ -352,27 +386,29 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
# ``response.release_conn()`` is called (implicitly by
# ``response.read()``)
except (Empty), e:
except Empty as e:
# Timed out by queue
raise TimeoutError("Request timed out. (pool_timeout=%s)" %
raise TimeoutError(self, "Request timed out. (pool_timeout=%s)" %
pool_timeout)
except (SocketTimeout), e:
except SocketTimeout as e:
# Timed out by socket
raise TimeoutError("Request timed out. (timeout=%s)" %
raise TimeoutError(self, "Request timed out. (timeout=%s)" %
timeout)
except (BaseSSLError), e:
except BaseSSLError as e:
# SSL certificate error
raise SSLError(e)
except (CertificateError), e:
except CertificateError as e:
# Name mismatch
raise SSLError(e)
except (HTTPException, SocketError), e:
except (HTTPException, SocketError) as e:
# Connection broken, discard. It will be replaced next _get_conn().
conn = None
# This is necessary so we can access e below
err = e
finally:
if conn and release_conn:
@@ -381,19 +417,16 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
if not conn:
log.warn("Retrying (%d attempts remain) after connection "
"broken by '%r': %s" % (retries, e, url))
"broken by '%r': %s" % (retries, err, url))
return self.urlopen(method, url, body, headers, retries - 1,
redirect, assert_same_host) # Try again
# Handle redirection
if (redirect and
response.status in [301, 302, 303, 307] and
'location' in response.headers): # Redirect, retry
log.info("Redirecting %s -> %s" %
(url, response.headers.get('location')))
return self.urlopen(method, response.headers.get('location'), body,
headers, retries - 1, redirect,
assert_same_host)
# Handle redirect?
redirect_location = redirect and response.get_redirect_location()
if redirect_location:
log.info("Redirecting %s -> %s" % (url, redirect_location))
return self.urlopen(method, redirect_location, body, headers,
retries - 1, redirect, assert_same_host)
return response
@@ -550,3 +583,22 @@ def connection_from_url(url, **kw):
return HTTPSConnectionPool(host, port=port, **kw)
else:
return HTTPConnectionPool(host, port=port, **kw)
def is_connection_dropped(conn):
"""
Returns True if the connection is dropped and should be closed.
:param conn:
``HTTPConnection`` object.
"""
if not poll:
return select([conn.sock], [], [], 0.0)[0]
# This version is better on platforms that support it.
p = poll()
p.register(conn.sock, POLLIN)
for (fno, ev) in p.poll(0.0):
if fno == conn.sock.fileno():
# Either data is buffered (bad), or the connection is dropped.
return True
+30 -11
View File
@@ -1,35 +1,54 @@
# urllib3/exceptions.py
# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
# Copyright 2008-2012 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
#
# This module is part of urllib3 and is released under
# the MIT License: http://www.opensource.org/licenses/mit-license.php
## Exceptions
## Base Exceptions
class HTTPError(Exception):
"Base exception used by this module."
pass
class SSLError(Exception):
class PoolError(HTTPError):
"Base exception for errors caused within a pool."
def __init__(self, pool, message):
self.pool = pool
HTTPError.__init__(self, "%s: %s" % (pool, message))
class SSLError(HTTPError):
"Raised when SSL certificate fails in an HTTPS connection."
pass
class MaxRetryError(HTTPError):
## Leaf Exceptions
class MaxRetryError(PoolError):
"Raised when the maximum number of retries is exceeded."
pass
def __init__(self, pool, url):
PoolError.__init__(self, pool,
"Max retries exceeded with url: %s" % url)
self.url = url
class TimeoutError(HTTPError):
class HostChangedError(PoolError):
"Raised when an existing pool gets a request for a foreign host."
def __init__(self, pool, url, retries=3):
PoolError.__init__(self, pool,
"Tried to open a foreign host with url: %s" % url)
self.url = url
self.retries = retries
class TimeoutError(PoolError):
"Raised when a socket timeout occurs."
pass
class HostChangedError(HTTPError):
"Raised when an existing pool gets a request for a foreign host."
pass
class EmptyPoolError(HTTPError):
class EmptyPoolError(PoolError):
"Raised when a pool runs out of connections and no more are allowed."
pass
+18 -15
View File
@@ -1,18 +1,21 @@
# urllib3/filepost.py
# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
# Copyright 2008-2012 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
#
# This module is part of urllib3 and is released under
# the MIT License: http://www.opensource.org/licenses/mit-license.php
import codecs
import mimetools
import mimetypes
try:
from cStringIO import StringIO
from mimetools import choose_boundary
except ImportError:
from StringIO import StringIO # pylint: disable-msg=W0404
from .packages.mimetools_choose_boundary import choose_boundary
from io import BytesIO
from .packages import six
from .packages.six import b
writer = codecs.lookup('utf-8')[3]
@@ -35,37 +38,37 @@ def encode_multipart_formdata(fields, boundary=None):
If not specified, then a random boundary will be generated using
:func:`mimetools.choose_boundary`.
"""
body = StringIO()
body = BytesIO()
if boundary is None:
boundary = mimetools.choose_boundary()
boundary = choose_boundary()
for fieldname, value in fields.iteritems():
body.write('--%s\r\n' % (boundary))
for fieldname, value in six.iteritems(fields):
body.write(b('--%s\r\n' % (boundary)))
if isinstance(value, tuple):
filename, data = value
writer(body).write('Content-Disposition: form-data; name="%s"; '
'filename="%s"\r\n' % (fieldname, filename))
body.write('Content-Type: %s\r\n\r\n' %
(get_content_type(filename)))
body.write(b('Content-Type: %s\r\n\r\n' %
(get_content_type(filename))))
else:
data = value
writer(body).write('Content-Disposition: form-data; name="%s"\r\n'
% (fieldname))
body.write('Content-Type: text/plain\r\n\r\n')
body.write(b'Content-Type: text/plain\r\n\r\n')
if isinstance(data, int):
data = str(data) # Backwards compatibility
if isinstance(data, unicode):
if isinstance(data, six.text_type):
writer(body).write(data)
else:
body.write(data)
body.write('\r\n')
body.write(b'\r\n')
body.write('--%s--\r\n' % (boundary))
body.write(b('--%s--\r\n' % (boundary)))
content_type = 'multipart/form-data; boundary=%s' % boundary
content_type = b('multipart/form-data; boundary=%s' % boundary)
return body.getvalue(), content_type
@@ -0,0 +1,47 @@
"""The function mimetools.choose_boundary() from Python 2.7, which seems to
have disappeared in Python 3 (although email.generator._make_boundary() might
work as a replacement?).
Tweaked to use lock from threading rather than thread.
"""
import os
from threading import Lock
_counter_lock = Lock()
_counter = 0
def _get_next_counter():
global _counter
with _counter_lock:
_counter += 1
return _counter
_prefix = None
def choose_boundary():
"""Return a string usable as a multipart boundary.
The string chosen is unique within a single program run, and
incorporates the user id (if available), process id (if available),
and current time. So it's very unlikely the returned string appears
in message text, but there's no guarantee.
The boundary contains dots so you have to quote it in the header."""
global _prefix
import time
if _prefix is None:
import socket
try:
hostid = socket.gethostbyname(socket.gethostname())
except socket.gaierror:
hostid = '127.0.0.1'
try:
uid = repr(os.getuid())
except AttributeError:
uid = '1'
try:
pid = repr(os.getpid())
except AttributeError:
pid = '1'
_prefix = hostid + '.' + uid + '.' + pid
return "%s.%.3f.%d" % (_prefix, time.time(), _get_next_counter())
@@ -0,0 +1,372 @@
"""Utilities for writing code that runs on Python 2 and 3"""
#Copyright (c) 2010-2011 Benjamin Peterson
#Permission is hereby granted, free of charge, to any person obtaining a copy of
#this software and associated documentation files (the "Software"), to deal in
#the Software without restriction, including without limitation the rights to
#use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
#the Software, and to permit persons to whom the Software is furnished to do so,
#subject to the following conditions:
#The above copyright notice and this permission notice shall be included in all
#copies or substantial portions of the Software.
#THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
#IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
#FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
#COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
#IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
#CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import operator
import sys
import types
__author__ = "Benjamin Peterson <benjamin@python.org>"
__version__ = "1.1.0"
# True if we are running on Python 3.
PY3 = sys.version_info[0] == 3
if PY3:
string_types = str,
integer_types = int,
class_types = type,
text_type = str
binary_type = bytes
MAXSIZE = sys.maxsize
else:
string_types = basestring,
integer_types = (int, long)
class_types = (type, types.ClassType)
text_type = unicode
binary_type = str
# It's possible to have sizeof(long) != sizeof(Py_ssize_t).
class X(object):
def __len__(self):
return 1 << 31
try:
len(X())
except OverflowError:
# 32-bit
MAXSIZE = int((1 << 31) - 1)
else:
# 64-bit
MAXSIZE = int((1 << 63) - 1)
del X
def _add_doc(func, doc):
"""Add documentation to a function."""
func.__doc__ = doc
def _import_module(name):
"""Import module, returning the module after the last dot."""
__import__(name)
return sys.modules[name]
class _LazyDescr(object):
def __init__(self, name):
self.name = name
def __get__(self, obj, tp):
result = self._resolve()
setattr(obj, self.name, result)
# This is a bit ugly, but it avoids running this again.
delattr(tp, self.name)
return result
class MovedModule(_LazyDescr):
def __init__(self, name, old, new=None):
super(MovedModule, self).__init__(name)
if PY3:
if new is None:
new = name
self.mod = new
else:
self.mod = old
def _resolve(self):
return _import_module(self.mod)
class MovedAttribute(_LazyDescr):
def __init__(self, name, old_mod, new_mod, old_attr=None, new_attr=None):
super(MovedAttribute, self).__init__(name)
if PY3:
if new_mod is None:
new_mod = name
self.mod = new_mod
if new_attr is None:
if old_attr is None:
new_attr = name
else:
new_attr = old_attr
self.attr = new_attr
else:
self.mod = old_mod
if old_attr is None:
old_attr = name
self.attr = old_attr
def _resolve(self):
module = _import_module(self.mod)
return getattr(module, self.attr)
class _MovedItems(types.ModuleType):
"""Lazy loading of moved objects"""
_moved_attributes = [
MovedAttribute("cStringIO", "cStringIO", "io", "StringIO"),
MovedAttribute("filter", "itertools", "builtins", "ifilter", "filter"),
MovedAttribute("map", "itertools", "builtins", "imap", "map"),
MovedAttribute("reload_module", "__builtin__", "imp", "reload"),
MovedAttribute("reduce", "__builtin__", "functools"),
MovedAttribute("StringIO", "StringIO", "io"),
MovedAttribute("xrange", "__builtin__", "builtins", "xrange", "range"),
MovedAttribute("zip", "itertools", "builtins", "izip", "zip"),
MovedModule("builtins", "__builtin__"),
MovedModule("configparser", "ConfigParser"),
MovedModule("copyreg", "copy_reg"),
MovedModule("http_cookiejar", "cookielib", "http.cookiejar"),
MovedModule("http_cookies", "Cookie", "http.cookies"),
MovedModule("html_entities", "htmlentitydefs", "html.entities"),
MovedModule("html_parser", "HTMLParser", "html.parser"),
MovedModule("http_client", "httplib", "http.client"),
MovedModule("BaseHTTPServer", "BaseHTTPServer", "http.server"),
MovedModule("CGIHTTPServer", "CGIHTTPServer", "http.server"),
MovedModule("SimpleHTTPServer", "SimpleHTTPServer", "http.server"),
MovedModule("cPickle", "cPickle", "pickle"),
MovedModule("queue", "Queue"),
MovedModule("reprlib", "repr"),
MovedModule("socketserver", "SocketServer"),
MovedModule("tkinter", "Tkinter"),
MovedModule("tkinter_dialog", "Dialog", "tkinter.dialog"),
MovedModule("tkinter_filedialog", "FileDialog", "tkinter.filedialog"),
MovedModule("tkinter_scrolledtext", "ScrolledText", "tkinter.scrolledtext"),
MovedModule("tkinter_simpledialog", "SimpleDialog", "tkinter.simpledialog"),
MovedModule("tkinter_tix", "Tix", "tkinter.tix"),
MovedModule("tkinter_constants", "Tkconstants", "tkinter.constants"),
MovedModule("tkinter_dnd", "Tkdnd", "tkinter.dnd"),
MovedModule("tkinter_colorchooser", "tkColorChooser",
"tkinter.colorchooser"),
MovedModule("tkinter_commondialog", "tkCommonDialog",
"tkinter.commondialog"),
MovedModule("tkinter_tkfiledialog", "tkFileDialog", "tkinter.filedialog"),
MovedModule("tkinter_font", "tkFont", "tkinter.font"),
MovedModule("tkinter_messagebox", "tkMessageBox", "tkinter.messagebox"),
MovedModule("tkinter_tksimpledialog", "tkSimpleDialog",
"tkinter.simpledialog"),
MovedModule("urllib_robotparser", "robotparser", "urllib.robotparser"),
MovedModule("winreg", "_winreg"),
]
for attr in _moved_attributes:
setattr(_MovedItems, attr.name, attr)
del attr
moves = sys.modules["six.moves"] = _MovedItems("moves")
def add_move(move):
"""Add an item to six.moves."""
setattr(_MovedItems, move.name, move)
def remove_move(name):
"""Remove item from six.moves."""
try:
delattr(_MovedItems, name)
except AttributeError:
try:
del moves.__dict__[name]
except KeyError:
raise AttributeError("no such move, %r" % (name,))
if PY3:
_meth_func = "__func__"
_meth_self = "__self__"
_func_code = "__code__"
_func_defaults = "__defaults__"
_iterkeys = "keys"
_itervalues = "values"
_iteritems = "items"
else:
_meth_func = "im_func"
_meth_self = "im_self"
_func_code = "func_code"
_func_defaults = "func_defaults"
_iterkeys = "iterkeys"
_itervalues = "itervalues"
_iteritems = "iteritems"
if PY3:
def get_unbound_function(unbound):
return unbound
advance_iterator = next
def callable(obj):
return any("__call__" in klass.__dict__ for klass in type(obj).__mro__)
else:
def get_unbound_function(unbound):
return unbound.im_func
def advance_iterator(it):
return it.next()
callable = callable
_add_doc(get_unbound_function,
"""Get the function out of a possibly unbound function""")
get_method_function = operator.attrgetter(_meth_func)
get_method_self = operator.attrgetter(_meth_self)
get_function_code = operator.attrgetter(_func_code)
get_function_defaults = operator.attrgetter(_func_defaults)
def iterkeys(d):
"""Return an iterator over the keys of a dictionary."""
return getattr(d, _iterkeys)()
def itervalues(d):
"""Return an iterator over the values of a dictionary."""
return getattr(d, _itervalues)()
def iteritems(d):
"""Return an iterator over the (key, value) pairs of a dictionary."""
return getattr(d, _iteritems)()
if PY3:
def b(s):
return s.encode("latin-1")
def u(s):
return s
if sys.version_info[1] <= 1:
def int2byte(i):
return bytes((i,))
else:
# This is about 2x faster than the implementation above on 3.2+
int2byte = operator.methodcaller("to_bytes", 1, "big")
import io
StringIO = io.StringIO
BytesIO = io.BytesIO
else:
def b(s):
return s
def u(s):
return unicode(s, "unicode_escape")
int2byte = chr
import StringIO
StringIO = BytesIO = StringIO.StringIO
_add_doc(b, """Byte literal""")
_add_doc(u, """Text literal""")
if PY3:
import builtins
exec_ = getattr(builtins, "exec")
def reraise(tp, value, tb=None):
if value.__traceback__ is not tb:
raise value.with_traceback(tb)
raise value
print_ = getattr(builtins, "print")
del builtins
else:
def exec_(code, globs=None, locs=None):
"""Execute code in a namespace."""
if globs is None:
frame = sys._getframe(1)
globs = frame.f_globals
if locs is None:
locs = frame.f_locals
del frame
elif locs is None:
locs = globs
exec("""exec code in globs, locs""")
exec_("""def reraise(tp, value, tb=None):
raise tp, value, tb
""")
def print_(*args, **kwargs):
"""The new-style print function."""
fp = kwargs.pop("file", sys.stdout)
if fp is None:
return
def write(data):
if not isinstance(data, basestring):
data = str(data)
fp.write(data)
want_unicode = False
sep = kwargs.pop("sep", None)
if sep is not None:
if isinstance(sep, unicode):
want_unicode = True
elif not isinstance(sep, str):
raise TypeError("sep must be None or a string")
end = kwargs.pop("end", None)
if end is not None:
if isinstance(end, unicode):
want_unicode = True
elif not isinstance(end, str):
raise TypeError("end must be None or a string")
if kwargs:
raise TypeError("invalid keyword arguments to print()")
if not want_unicode:
for arg in args:
if isinstance(arg, unicode):
want_unicode = True
break
if want_unicode:
newline = unicode("\n")
space = unicode(" ")
else:
newline = "\n"
space = " "
if sep is None:
sep = space
if end is None:
end = newline
for i, arg in enumerate(args):
if i:
write(sep)
write(arg)
write(end)
_add_doc(reraise, """Reraise an exception.""")
def with_metaclass(meta, base=object):
"""Create a base class with a metaclass."""
return meta("NewBase", (base,), {})
+14 -14
View File
@@ -1,32 +1,27 @@
# urllib3/poolmanager.py
# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
# Copyright 2008-2012 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
#
# This module is part of urllib3 and is released under
# the MIT License: http://www.opensource.org/licenses/mit-license.php
import logging
from ._collections import RecentlyUsedContainer
from .connectionpool import (
HTTPConnectionPool, HTTPSConnectionPool,
get_host, connection_from_url,
)
from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool
from .connectionpool import get_host, connection_from_url, port_by_scheme
from .exceptions import HostChangedError
from .request import RequestMethods
__all__ = ['PoolManager', 'ProxyManager', 'proxy_from_url']
from .request import RequestMethods
from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool
pool_classes_by_scheme = {
'http': HTTPConnectionPool,
'https': HTTPSConnectionPool,
}
port_by_scheme = {
'http': 80,
'https': 443,
}
log = logging.getLogger(__name__)
class PoolManager(RequestMethods):
@@ -105,7 +100,12 @@ class PoolManager(RequestMethods):
:class:`urllib3.connectionpool.ConnectionPool` can be chosen for it.
"""
conn = self.connection_from_url(url)
return conn.urlopen(method, url, assert_same_host=False, **kw)
try:
return conn.urlopen(method, url, **kw)
except HostChangedError as e:
kw['retries'] = e.retries # Persist retries countdown
return self.urlopen(method, e.url, **kw)
class ProxyManager(RequestMethods):
+5 -3
View File
@@ -1,11 +1,13 @@
# urllib3/request.py
# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
# Copyright 2008-2012 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
#
# This module is part of urllib3 and is released under
# the MIT License: http://www.opensource.org/licenses/mit-license.php
from urllib import urlencode
try:
from urllib.parse import urlencode
except ImportError:
from urllib import urlencode
from .filepost import encode_multipart_formdata
+50 -35
View File
@@ -1,5 +1,5 @@
# urllib3/response.py
# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
# Copyright 2008-2012 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
#
# This module is part of urllib3 and is released under
# the MIT License: http://www.opensource.org/licenses/mit-license.php
@@ -8,21 +8,22 @@ import gzip
import logging
import zlib
try:
from cStringIO import StringIO
except ImportError:
from StringIO import StringIO # pylint: disable-msg=W0404
from io import BytesIO
from .exceptions import HTTPError
try:
basestring = basestring
except NameError: # Python 3
basestring = (str, bytes)
log = logging.getLogger(__name__)
def decode_gzip(data):
gzipper = gzip.GzipFile(fileobj=StringIO(data))
gzipper = gzip.GzipFile(fileobj=BytesIO(data))
return gzipper.read()
@@ -71,7 +72,7 @@ class HTTPResponse(object):
self.strict = strict
self._decode_content = decode_content
self._body = None
self._body = body if body and isinstance(body, basestring) else None
self._fp = None
self._original_response = original_response
@@ -81,9 +82,22 @@ class HTTPResponse(object):
if hasattr(body, 'read'):
self._fp = body
if preload_content:
if preload_content and not self._body:
self._body = self.read(decode_content=decode_content)
def get_redirect_location(self):
"""
Should we redirect and where to?
:returns: Truthy redirect location string if we got a redirect status
code and valid location. ``None`` if redirect status and no
location. ``False`` if not a redirect status code.
"""
if self.status in [301, 302, 303, 307]:
return self.headers.get('location')
return False
def release_conn(self):
if not self._pool or not self._connection:
return
@@ -98,10 +112,9 @@ class HTTPResponse(object):
return self._body
if self._fp:
return self.read(decode_content=self._decode_content,
cache_content=True)
return self.read(cache_content=True)
def read(self, amt=None, decode_content=True, cache_content=False):
def read(self, amt=None, decode_content=None, cache_content=False):
"""
Similar to :meth:`httplib.HTTPResponse.read`, but with two additional
parameters: ``decode_content`` and ``cache_content``.
@@ -124,22 +137,22 @@ class HTTPResponse(object):
"""
content_encoding = self.headers.get('content-encoding')
decoder = self.CONTENT_DECODERS.get(content_encoding)
if decode_content is None:
decode_content = self._decode_content
data = self._fp and self._fp.read(amt)
if self._fp is None:
return
try:
if amt:
return data
if not decode_content or not decoder:
if cache_content:
self._body = data
return data
if amt is None:
# cStringIO doesn't like amt=None
data = self._fp.read()
else:
return self._fp.read(amt)
try:
data = decoder(data)
if decode_content and decoder:
data = decoder(data)
except IOError:
raise HTTPError("Received response with content-encoding: %s, but "
"failed to decode it." % content_encoding)
@@ -150,12 +163,11 @@ class HTTPResponse(object):
return data
finally:
if self._original_response and self._original_response.isclosed():
self.release_conn()
@staticmethod
def from_httplib(r, **response_kw):
@classmethod
def from_httplib(ResponseCls, r, **response_kw):
"""
Given an :class:`httplib.HTTPResponse` instance ``r``, return a
corresponding :class:`urllib3.response.HTTPResponse` object.
@@ -164,14 +176,17 @@ class HTTPResponse(object):
with ``original_response=r``.
"""
return HTTPResponse(body=r,
headers=dict(r.getheaders()),
status=r.status,
version=r.version,
reason=r.reason,
strict=r.strict,
original_response=r,
**response_kw)
# HTTPResponse objects in Python 3 don't have a .strict attribute
strict = getattr(r, 'strict', 0)
return ResponseCls(body=r,
# In Python 3, the header keys are returned capitalised
headers=dict((k.lower(), v) for k,v in r.getheaders()),
status=r.status,
version=r.version,
reason=r.reason,
strict=strict,
original_response=r,
**response_kw)
# Backwards-compatibility methods for httplib.HTTPResponse
def getheaders(self):