dinislage/CouchPotatoServer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Packages update

Browse Source
This commit is contained in:
Ruud
2012-02-11 16:28:06 +01:00
parent 3bbf1126c3
commit 02e01fb2d6
217 changed files with 26395 additions and 21194 deletions
Download Patch File Download Diff File Expand all files Collapse all files
libs/requests/__init__.py
+2 -2
View File
@@ -15,8 +15,8 @@ requests
"""
__title__ = 'requests'
__version__ = '0.9.1'
__build__ = 0x000901
__version__ = '0.10.1'
__build__ = 0x001001
__author__ = 'Kenneth Reitz'
__license__ = 'ISC'
__copyright__ = 'Copyright 2012 Kenneth Reitz'
libs/requests/api.py
+2 -1
View File
@@ -32,9 +32,10 @@ def request(method, url, **kwargs):
:param session: (optional) A :class:`Session` object to be used for the request.
:param config: (optional) A configuration dictionary.
:param verify: (optional) if ``True``, the SSL cert will be verified. A CA_BUNDLE path can also be provided.
:param prefetch: (optional) if ``True``, the response content will be immediately downloaded.
"""
s = kwargs.get('session') or sessions.session()
s = kwargs.pop('session') if 'session' in kwargs else sessions.session()
return s.request(method=method, url=url, **kwargs)
libs/requests/async.py
+11 -19
View File
@@ -46,15 +46,15 @@ def patched(f):
return wrapped
def send(r, pools=None):
"""Sends a given Request object."""
def send(r, pool=None):
"""Sends the request object using the specified pool. If a pool isn't
specified this method blocks. Pools are useful because you can specify size
and can hence limit concurrency."""
if pools:
r._pools = pools
if pool != None:
return pool.spawn(r.send)
r.send()
return r.response
return gevent.spawn(r.send)
# Patched requests.api functions.
@@ -78,19 +78,11 @@ def map(requests, prefetch=True, size=None):
requests = list(requests)
if size:
pool = Pool(size)
pool.map(send, requests)
pool.join()
else:
jobs = [gevent.spawn(send, r) for r in requests]
gevent.joinall(jobs)
pool = Pool(size) if size else None
jobs = [send(r, pool) for r in requests]
gevent.joinall(jobs)
if prefetch:
[r.response.content for r in requests]
return [r.response for r in requests]
return [r.response for r in requests]
libs/requests/auth.py
+27 -13
View File
@@ -7,19 +7,21 @@ requests.auth
This module contains the authentication handlers for Requests.
"""
from __future__ import unicode_literals
import time
import hashlib
from base64 import b64encode
from urlparse import urlparse
from .compat import urlparse, str, bytes
from .utils import randombytes, parse_dict_header
def _basic_auth_str(username, password):
"""Returns a Basic Auth string."""
return 'Basic %s' % b64encode('%s:%s' % (username, password))
return 'Basic ' + b64encode(("%s:%s" % (username, password)).encode('utf-8')).strip().decode('utf-8')
class AuthBase(object):
@@ -32,8 +34,8 @@ class AuthBase(object):
class HTTPBasicAuth(AuthBase):
"""Attaches HTTP Basic Authentication to the given Request object."""
def __init__(self, username, password):
self.username = str(username)
self.password = str(password)
self.username = username
self.password = password
def __call__(self, r):
r.headers['Authorization'] = _basic_auth_str(self.username, self.password)
@@ -74,9 +76,17 @@ class HTTPDigestAuth(AuthBase):
algorithm = algorithm.upper()
# lambdas assume digest modules are imported at the top level
if algorithm == 'MD5':
H = lambda x: hashlib.md5(x).hexdigest()
def h(x):
if isinstance(x, str):
x = x.encode('utf-8')
return hashlib.md5(x).hexdigest()
H = h
elif algorithm == 'SHA':
H = lambda x: hashlib.sha1(x).hexdigest()
def h(x):
if isinstance(x, str):
x = x.encode('utf-8')
return hashlib.sha1(x).hexdigest()
H = h
# XXX MD5-sess
KD = lambda s, d: H("%s:%s" % (s, d))
@@ -86,7 +96,9 @@ class HTTPDigestAuth(AuthBase):
# XXX not implemented yet
entdig = None
p_parsed = urlparse(r.request.url)
path = p_parsed.path + p_parsed.query
path = p_parsed.path
if p_parsed.query:
path += '?' + p_parsed.query
A1 = '%s:%s:%s' % (self.username, realm, self.password)
A2 = '%s:%s' % (r.request.method, path)
@@ -99,10 +111,12 @@ class HTTPDigestAuth(AuthBase):
last_nonce = nonce
ncvalue = '%08x' % nonce_count
cnonce = (hashlib.sha1("%s:%s:%s:%s" % (
nonce_count, nonce, time.ctime(), randombytes(8)))
.hexdigest()[:16]
)
s = str(nonce_count).encode('utf-8')
s += nonce.encode('utf-8')
s += time.ctime().encode('utf-8')
s += randombytes(8)
cnonce = (hashlib.sha1(s).hexdigest()[:16])
noncebit = "%s:%s:%s:%s:%s" % (nonce, ncvalue, cnonce, qop, H(A2))
respdig = KD(H(A1), noncebit)
elif qop is None:
@@ -132,5 +146,5 @@ class HTTPDigestAuth(AuthBase):
return r
def __call__(self, r):
r.hooks['response'] = self.handle_401
r.register_hook('response', self.handle_401)
return r
libs/requests/compat.py
+102
View File
@@ -0,0 +1,102 @@
# -*- coding: utf-8 -*-
"""
pythoncompat
"""
import sys
# -------
# Pythons
# -------
# Syntax sugar.
_ver = sys.version_info
#: Python 2.x?
is_py2 = (_ver[0] == 2)
#: Python 3.x?
is_py3 = (_ver[0] == 3)
#: Python 3.0.x
is_py30 = (is_py3 and _ver[1] == 0)
#: Python 3.1.x
is_py31 = (is_py3 and _ver[1] == 1)
#: Python 3.2.x
is_py32 = (is_py3 and _ver[1] == 2)
#: Python 3.3.x
is_py33 = (is_py3 and _ver[1] == 3)
#: Python 3.4.x
is_py34 = (is_py3 and _ver[1] == 4)
#: Python 2.7.x
is_py27 = (is_py2 and _ver[1] == 7)
#: Python 2.6.x
is_py26 = (is_py2 and _ver[1] == 6)
#: Python 2.5.x
is_py25 = (is_py2 and _ver[1] == 5)
#: Python 2.4.x
is_py24 = (is_py2 and _ver[1] == 4) # I'm assuming this is not by choice.
# ---------
# Platforms
# ---------
# Syntax sugar.
_ver = sys.version.lower()
is_pypy = ('pypy' in _ver)
is_jython = ('jython' in _ver)
is_ironpython = ('iron' in _ver)
# Assume CPython, if nothing else.
is_cpython = not any((is_pypy, is_jython, is_ironpython))
# Windows-based system.
is_windows = 'win32' in str(sys.platform).lower()
# Standard Linux 2+ system.
is_linux = ('linux' in str(sys.platform).lower())
is_osx = ('darwin' in str(sys.platform).lower())
is_hpux = ('hpux' in str(sys.platform).lower()) # Complete guess.
is_solaris = ('solar==' in str(sys.platform).lower()) # Complete guess.
# ---------
# Specifics
# ---------
if is_py2:
from urllib import quote, unquote, urlencode
from urlparse import urlparse, urlunparse, urljoin, urlsplit
from urllib2 import parse_http_list
import cookielib
from .packages.oreos.monkeys import SimpleCookie
from StringIO import StringIO
str = unicode
bytes = str
elif is_py3:
from urllib.parse import urlparse, urlunparse, urljoin, urlsplit, urlencode, quote, unquote
from urllib.request import parse_http_list
from http import cookiejar as cookielib
from http.cookies import SimpleCookie
from io import StringIO
str = str
bytes = bytes
libs/requests/defaults.py
+1 -5
View File
@@ -10,16 +10,13 @@ Configurations:
:base_headers: Default HTTP headers.
:verbose: Stream to write request logging to.
:timeout: Seconds until request timeout.
:max_redirects: Maximum number of redirects allowed within a request.
:decode_unicode: Decode unicode responses automatically?
:max_redirects: Maximum number of redirects allowed within a request.s
:keep_alive: Reuse HTTP Connections?
:max_retries: The number of times a request should be retried in the event of a connection failure.
:danger_mode: If true, Requests will raise errors immediately.
:safe_mode: If true, Requests will catch all errors.
:pool_maxsize: The maximium size of an HTTP connection pool.
:pool_connections: The number of active HTTP connection pools to use.
"""
from . import __version__
@@ -35,7 +32,6 @@ defaults['base_headers'] = {
defaults['verbose'] = None
defaults['max_redirects'] = 30
defaults['decode_unicode'] = True
defaults['pool_connections'] = 10
defaults['pool_maxsize'] = 10
defaults['max_retries'] = 0
libs/requests/hooks.py
+13 -5
View File
@@ -22,7 +22,10 @@ Available hooks:
"""
import warnings
import traceback
HOOKS = ('args', 'pre_request', 'post_request', 'response')
def dispatch_hook(key, hooks, hook_data):
@@ -31,10 +34,15 @@ def dispatch_hook(key, hooks, hook_data):
hooks = hooks or dict()
if key in hooks:
try:
return hooks.get(key).__call__(hook_data) or hook_data
hooks = hooks.get(key)
except Exception, why:
warnings.warn(str(why))
if hasattr(hooks, '__call__'):
hooks = [hooks]
for hook in hooks:
try:
hook_data = hook(hook_data) or hook_data
except Exception:
traceback.print_exc()
return hook_data
libs/requests/models.py
+157 -61
View File
@@ -8,15 +8,12 @@ This module contains the primary objects that power Requests.
"""
import os
import urllib
from urlparse import urlparse, urlunparse, urljoin, urlsplit
from datetime import datetime
from .hooks import dispatch_hook
from .hooks import dispatch_hook, HOOKS
from .structures import CaseInsensitiveDict
from .status_codes import codes
from .packages import oreos
from .auth import HTTPBasicAuth, HTTPProxyAuth
from .packages.urllib3.response import HTTPResponse
from .packages.urllib3.exceptions import MaxRetryError
@@ -29,8 +26,15 @@ from .exceptions import (
URLRequired, SSLError)
from .utils import (
get_encoding_from_headers, stream_decode_response_unicode,
stream_decompress, guess_filename, requote_path)
stream_decompress, guess_filename, requote_path, dict_from_string)
from .compat import urlparse, urlunparse, urljoin, urlsplit, urlencode, quote, unquote, str, bytes, SimpleCookie, is_py3, is_py2
# Import chardet if it is available.
try:
import chardet
except ImportError:
pass
REDIRECT_STATI = (codes.moved, codes.found, codes.other, codes.temporary_moved)
@@ -57,13 +61,19 @@ class Request(object):
hooks=None,
config=None,
_poolmanager=None,
verify=None):
verify=None,
session=None):
#: Float describes the timeout of the request.
# (Use socket.setdefaulttimeout() as fallback)
self.timeout = timeout
#: Request URL.
# if isinstance(url, str):
# url = url.encode('utf-8')
# print(dir(url))
self.url = url
#: Dictionary of HTTP Headers to attach to the :class:`Request <Request>`.
@@ -82,7 +92,6 @@ class Request(object):
#: Dictionary or byte of querystring data to attach to the
#: :class:`Request <Request>`.
self.params = None
self.params = dict(params or [])
#: True if :class:`Request <Request>` is part of a redirect chain (disables history
#: and HTTPError storage).
@@ -114,10 +123,18 @@ class Request(object):
self.sent = False
#: Event-handling hooks.
self.hooks = hooks
self.hooks = {}
for event in HOOKS:
self.hooks[event] = []
hooks = hooks or {}
for (k, v) in list(hooks.items()):
self.register_hook(event=k, hook=v)
#: Session.
self.session = None
self.session = session
#: SSL Verification.
self.verify = verify
@@ -128,7 +145,7 @@ class Request(object):
headers = CaseInsensitiveDict()
# Add configured base headers.
for (k, v) in self.config.get('base_headers', {}).items():
for (k, v) in list(self.config.get('base_headers', {}).items()):
if k not in headers:
headers[k] = v
@@ -144,7 +161,7 @@ class Request(object):
return '<Request [%s]>' % (self.method)
def _build_response(self, resp, is_error=False):
def _build_response(self, resp):
"""Build internal :class:`Response <Response>` object
from given response.
"""
@@ -173,7 +190,7 @@ class Request(object):
# Add new cookies from the server.
if 'set-cookie' in response.headers:
cookie_header = response.headers['set-cookie']
cookies = oreos.dict_from_string(cookie_header)
cookies = dict_from_string(cookie_header)
# Save cookies in Response.
response.cookies = cookies
@@ -183,10 +200,6 @@ class Request(object):
# Save original response for later.
response.raw = resp
if is_error:
response.error = resp
response.url = self.full_url
return response
@@ -247,7 +260,8 @@ class Request(object):
timeout=self.timeout,
_poolmanager=self._poolmanager,
proxies = self.proxies,
verify = self.verify
verify = self.verify,
session = self.session
)
request.send()
@@ -274,16 +288,17 @@ class Request(object):
returns it twice.
"""
if hasattr(data, '__iter__'):
if hasattr(data, '__iter__') and not isinstance(data, str):
data = dict(data)
if hasattr(data, 'items'):
result = []
for k, vs in data.items():
for k, vs in list(data.items()):
for v in isinstance(vs, list) and vs or [vs]:
result.append((k.encode('utf-8') if isinstance(k, unicode) else k,
v.encode('utf-8') if isinstance(v, unicode) else v))
return result, urllib.urlencode(result, doseq=True)
result.append((k.encode('utf-8') if isinstance(k, str) else k,
v.encode('utf-8') if isinstance(v, str) else v))
return result, urlencode(result, doseq=True)
else:
return data, data
@@ -294,20 +309,27 @@ class Request(object):
if not self.url:
raise URLRequired()
url = self.url
# Support for unicode domain names and paths.
scheme, netloc, path, params, query, fragment = urlparse(self.url)
scheme, netloc, path, params, query, fragment = urlparse(url)
if not scheme:
raise ValueError()
raise ValueError("Invalid URL %r: No schema supplied" % url)
netloc = netloc.encode('idna')
netloc = netloc.encode('idna').decode('utf-8')
if isinstance(path, unicode):
path = path.encode('utf-8')
if is_py2:
if isinstance(path, str):
path = path.encode('utf-8')
path = requote_path(path)
path = requote_path(path)
url = str(urlunparse([ scheme, netloc, path, params, query, fragment ]))
# print([ scheme, netloc, path, params, query, fragment ])
# print('---------------------')
url = (urlunparse([ scheme, netloc, path, params, query, fragment ]))
if self._enc_params:
if urlparse(url).query:
@@ -332,6 +354,10 @@ class Request(object):
path = p.path
if not path:
path = '/'
# if is_py3:
path = quote(path.encode('utf-8'))
url.append(path)
query = p.query
@@ -339,9 +365,16 @@ class Request(object):
url.append('?')
url.append(query)
# print(url)
return ''.join(url)
def register_hook(self, event, hook):
"""Properly register a hook."""
return self.hooks[event].append(hook)
def send(self, anyway=False, prefetch=False):
"""Sends the request. Returns True of successful, false if not.
@@ -369,14 +402,14 @@ class Request(object):
# Multi-part file uploads.
if self.files:
if not isinstance(self.data, basestring):
if not isinstance(self.data, str):
try:
fields = self.data.copy()
except AttributeError:
fields = dict(self.data)
for (k, v) in self.files.items():
for (k, v) in list(self.files.items()):
# support for explicit filename
if isinstance(v, (tuple, list)):
fn, fp = v
@@ -393,7 +426,7 @@ class Request(object):
if self.data:
body = self._enc_data
if isinstance(self.data, basestring):
if isinstance(self.data, str):
content_type = None
else:
content_type = 'application/x-www-form-urlencoded'
@@ -454,6 +487,9 @@ class Request(object):
conn.cert_reqs = 'CERT_REQUIRED'
conn.ca_certs = cert_loc
else:
conn.cert_reqs = 'CERT_NONE'
conn.ca_certs = None
if not self.sent or anyway:
@@ -463,8 +499,8 @@ class Request(object):
if 'cookie' not in self.headers:
# Simple cookie with our dict.
c = oreos.monkeys.SimpleCookie()
for (k, v) in self.cookies.items():
c = SimpleCookie()
for (k, v) in list(self.cookies.items()):
c[k] = v
# Turn it into a header.
@@ -493,16 +529,16 @@ class Request(object):
)
self.sent = True
except MaxRetryError, e:
except MaxRetryError as e:
raise ConnectionError(e)
except (_SSLError, _HTTPError), e:
except (_SSLError, _HTTPError) as e:
if self.verify and isinstance(e, _SSLError):
raise SSLError(e)
raise Timeout('Request timed out.')
except RequestException, e:
except RequestException as e:
if self.config.get('safe_mode', False):
# In safe mode, catch the exception and attach it to
# a blank urllib3.HTTPResponse object.
@@ -524,7 +560,7 @@ class Request(object):
if prefetch:
# Save the response.
self.response.content
if self.config.get('danger_mode'):
self.response.raise_for_status()
@@ -581,6 +617,10 @@ class Response(object):
def __repr__(self):
return '<Response [%s]>' % (self.status_code)
def __bool__(self):
"""Returns true if :attr:`status_code` is 'OK'."""
return self.ok
def __nonzero__(self):
"""Returns true if :attr:`status_code` is 'OK'."""
return self.ok
@@ -594,7 +634,7 @@ class Response(object):
return True
def iter_content(self, chunk_size=10 * 1024, decode_unicode=None):
def iter_content(self, chunk_size=10 * 1024, decode_unicode=False):
"""Iterates over the response data. This avoids reading the content
at once into memory for large responses. The chunk size is the number
of bytes it should read into memory. This is not necessarily the
@@ -613,16 +653,41 @@ class Response(object):
yield chunk
self._content_consumed = True
gen = generate()
def generate_chunked():
resp = self.raw._original_response
fp = resp.fp
if resp.chunk_left is not None:
pending_bytes = resp.chunk_left
while pending_bytes:
chunk = fp.read(min(chunk_size, pending_bytes))
pending_bytes-=len(chunk)
yield chunk
fp.read(2) # throw away crlf
while 1:
#XXX correct line size? (httplib has 64kb, seems insane)
pending_bytes = fp.readline(40).strip()
pending_bytes = int(pending_bytes, 16)
if pending_bytes == 0:
break
while pending_bytes:
chunk = fp.read(min(chunk_size, pending_bytes))
pending_bytes-=len(chunk)
yield chunk
fp.read(2) # throw away crlf
self._content_consumed = True
fp.close()
if getattr(getattr(self.raw, '_original_response', None), 'chunked', False):
gen = generate_chunked()
else:
gen = generate()
if 'gzip' in self.headers.get('content-encoding', ''):
gen = stream_decompress(gen, mode='gzip')
elif 'deflate' in self.headers.get('content-encoding', ''):
gen = stream_decompress(gen, mode='deflate')
if decode_unicode is None:
decode_unicode = self.config.get('decode_unicode')
if decode_unicode:
gen = stream_decode_response_unicode(gen, self)
@@ -635,15 +700,29 @@ class Response(object):
responses.
"""
#TODO: why rstrip by default
pending = None
for chunk in self.iter_content(chunk_size, decode_unicode=decode_unicode):
if pending is not None:
chunk = pending + chunk
lines = chunk.splitlines(True)
for line in lines[:-1]:
yield line.rstrip()
# Save the last part of the chunk for next iteration, to keep full line together
pending = lines[-1]
# lines may be empty for the last chunk of a chunked response
if lines:
pending = lines[-1]
#if pending is a complete line, give it baack
if pending[-1] == '\n':
yield pending.rstrip()
pending = None
else:
pending = None
# Yield the last line
if pending is not None:
@@ -652,9 +731,7 @@ class Response(object):
@property
def content(self):
"""Content of the response, in bytes or unicode
(if available).
"""
"""Content of the response, in bytes."""
if self._content is None:
# Read the contents.
@@ -667,26 +744,45 @@ class Response(object):
except AttributeError:
self._content = None
content = self._content
self._content_consumed = True
return self._content
# Decode unicode content.
if self.config.get('decode_unicode'):
# Try charset from content-type
@property
def text(self):
"""Content of the response, in unicode.
if self.encoding:
try:
content = unicode(content, self.encoding)
except UnicodeError:
pass
if Response.encoding is None and chardet module is available, encoding
will be guessed.
"""
# Fall back:
# Try charset from content-type
content = None
encoding = self.encoding
# Fallback to auto-detected encoding if chardet is available.
if self.encoding is None:
try:
content = unicode(content, self.encoding, errors='replace')
except TypeError:
detected = chardet.detect(self.content) or {}
encoding = detected.get('encoding')
# Trust that chardet isn't available or something went terribly wrong.
except Exception:
pass
# Decode unicode from given encoding.
try:
content = str(self.content, encoding)
except (UnicodeError, TypeError):
pass
# Try to fall back:
if not content:
try:
content = str(content, encoding, errors='replace')
except (UnicodeError, TypeError):
pass
self._content_consumed = True
return content
libs/requests/packages/oreos/monkeys.py
+1 -1
View File
@@ -318,7 +318,7 @@ _Translator = {
'\375' : '\\375', '\376' : '\\376', '\377' : '\\377'
}
_idmap = ''.join(chr(x) for x in xrange(256))
_idmap = ''.join(chr(x) for x in range(256))
def _quote(str, LegalChars=_LegalChars,
idmap=_idmap, translate=string.translate):
libs/requests/packages/urllib3/__init__.py
+2 -2
View File
@@ -1,5 +1,5 @@
# urllib3/__init__.py
# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
# Copyright 2008-2012 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
#
# This module is part of urllib3 and is released under
# the MIT License: http://www.opensource.org/licenses/mit-license.php
@@ -10,7 +10,7 @@ urllib3 - Thread-safe connection pooling and re-using.
__author__ = 'Andrey Petrov (andrey.petrov@shazow.net)'
__license__ = 'MIT'
__version__ = '1.0.2'
__version__ = '1.1'
from .connectionpool import (
libs/requests/packages/urllib3/_collections.py
+1 -1
View File
@@ -1,5 +1,5 @@
# urllib3/_collections.py
# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
# Copyright 2008-2012 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
#
# This module is part of urllib3 and is released under
# the MIT License: http://www.opensource.org/licenses/mit-license.php
libs/requests/packages/urllib3/connectionpool.py
+88 -36
View File
@@ -1,5 +1,5 @@
# urllib3/connectionpool.py
# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
# Copyright 2008-2012 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
#
# This module is part of urllib3 and is released under
# the MIT License: http://www.opensource.org/licenses/mit-license.php
@@ -7,15 +7,27 @@
import logging
import socket
from httplib import HTTPConnection, HTTPSConnection, HTTPException
from Queue import Queue, Empty, Full
from select import select
from socket import error as SocketError, timeout as SocketTimeout
from .packages.ssl_match_hostname import match_hostname, CertificateError
try:
from select import poll, POLLIN
except ImportError: # Doesn't exist on OSX and other platforms
from select import select
poll = False
try: # Python 3
from http.client import HTTPConnection, HTTPSConnection, HTTPException
from http.client import HTTP_PORT, HTTPS_PORT
except ImportError:
from httplib import HTTPConnection, HTTPSConnection, HTTPException
from httplib import HTTP_PORT, HTTPS_PORT
try: # Python 3
from queue import Queue, Empty, Full
except ImportError:
from Queue import Queue, Empty, Full
try: # Compiled with SSL?
import ssl
BaseSSLError = ssl.SSLError
except ImportError:
@@ -23,21 +35,29 @@ except ImportError:
BaseSSLError = None
from .packages.ssl_match_hostname import match_hostname, CertificateError
from .request import RequestMethods
from .response import HTTPResponse
from .exceptions import (
SSLError,
from .exceptions import (SSLError,
MaxRetryError,
TimeoutError,
HostChangedError,
EmptyPoolError,
)
from .packages.ssl_match_hostname import match_hostname, CertificateError
from .packages import six
xrange = six.moves.xrange
log = logging.getLogger(__name__)
_Default = object()
port_by_scheme = {
'http': HTTP_PORT,
'https': HTTPS_PORT,
}
## Connection objects (extension of httplib)
@@ -81,7 +101,16 @@ class ConnectionPool(object):
Base class for all connection pools, such as
:class:`.HTTPConnectionPool` and :class:`.HTTPSConnectionPool`.
"""
pass
scheme = None
def __init__(self, host, port=None):
self.host = host
self.port = port
def __str__(self):
return '%s(host=%r, port=%r)' % (type(self).__name__,
self.host, self.port)
class HTTPConnectionPool(ConnectionPool, RequestMethods):
@@ -169,14 +198,14 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
conn = self.pool.get(block=self.block, timeout=timeout)
# If this is a persistent connection, check if it got disconnected
if conn and conn.sock and select([conn.sock], [], [], 0.0)[0]:
# Either data is buffered (bad), or the connection is dropped.
if conn and conn.sock and is_connection_dropped(conn):
log.info("Resetting dropped connection: %s" % self.host)
conn.close()
except Empty:
if self.block:
raise EmptyPoolError("Pool reached maximum size and no more "
raise EmptyPoolError(self,
"Pool reached maximum size and no more "
"connections are allowed.")
pass # Oh well, we'll create a new connection then
@@ -229,11 +258,17 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
def is_same_host(self, url):
"""
Check if the given ``url`` is a member of the same host as this
conncetion pool.
connection pool.
"""
# TODO: Add optional support for socket.gethostbyname checking.
scheme, host, port = get_host(url)
if self.port and not port:
# Use explicit default port for comparison when none is given.
port = port_by_scheme.get(scheme)
return (url.startswith('/') or
get_host(url) == (self.scheme, self.host, self.port))
(scheme, host, port) == (self.scheme, self.host, self.port))
def urlopen(self, method, url, body=None, headers=None, retries=3,
redirect=True, assert_same_host=True, timeout=_Default,
@@ -306,7 +341,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
headers = self.headers
if retries < 0:
raise MaxRetryError("Max retries exceeded for url: %s" % url)
raise MaxRetryError(self, url)
if timeout is _Default:
timeout = self.timeout
@@ -320,8 +355,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
if self.port:
host = "%s:%d" % (host, self.port)
raise HostChangedError("Connection pool with host '%s' tried to "
"open a foreign host: %s" % (host, url))
raise HostChangedError(self, url, retries - 1)
conn = None
@@ -352,27 +386,29 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
# ``response.release_conn()`` is called (implicitly by
# ``response.read()``)
except (Empty), e:
except Empty as e:
# Timed out by queue
raise TimeoutError("Request timed out. (pool_timeout=%s)" %
raise TimeoutError(self, "Request timed out. (pool_timeout=%s)" %
pool_timeout)
except (SocketTimeout), e:
except SocketTimeout as e:
# Timed out by socket
raise TimeoutError("Request timed out. (timeout=%s)" %
raise TimeoutError(self, "Request timed out. (timeout=%s)" %
timeout)
except (BaseSSLError), e:
except BaseSSLError as e:
# SSL certificate error
raise SSLError(e)
except (CertificateError), e:
except CertificateError as e:
# Name mismatch
raise SSLError(e)
except (HTTPException, SocketError), e:
except (HTTPException, SocketError) as e:
# Connection broken, discard. It will be replaced next _get_conn().
conn = None
# This is necessary so we can access e below
err = e
finally:
if conn and release_conn:
@@ -381,19 +417,16 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
if not conn:
log.warn("Retrying (%d attempts remain) after connection "
"broken by '%r': %s" % (retries, e, url))
"broken by '%r': %s" % (retries, err, url))
return self.urlopen(method, url, body, headers, retries - 1,
redirect, assert_same_host) # Try again
# Handle redirection
if (redirect and
response.status in [301, 302, 303, 307] and
'location' in response.headers): # Redirect, retry
log.info("Redirecting %s -> %s" %
(url, response.headers.get('location')))
return self.urlopen(method, response.headers.get('location'), body,
headers, retries - 1, redirect,
assert_same_host)
# Handle redirect?
redirect_location = redirect and response.get_redirect_location()
if redirect_location:
log.info("Redirecting %s -> %s" % (url, redirect_location))
return self.urlopen(method, redirect_location, body, headers,
retries - 1, redirect, assert_same_host)
return response
@@ -550,3 +583,22 @@ def connection_from_url(url, **kw):
return HTTPSConnectionPool(host, port=port, **kw)
else:
return HTTPConnectionPool(host, port=port, **kw)
def is_connection_dropped(conn):
"""
Returns True if the connection is dropped and should be closed.
:param conn:
``HTTPConnection`` object.
"""
if not poll:
return select([conn.sock], [], [], 0.0)[0]
# This version is better on platforms that support it.
p = poll()
p.register(conn.sock, POLLIN)
for (fno, ev) in p.poll(0.0):
if fno == conn.sock.fileno():
# Either data is buffered (bad), or the connection is dropped.
return True
libs/requests/packages/urllib3/exceptions.py
+30 -11
View File
@@ -1,35 +1,54 @@
# urllib3/exceptions.py
# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
# Copyright 2008-2012 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
#
# This module is part of urllib3 and is released under
# the MIT License: http://www.opensource.org/licenses/mit-license.php
## Exceptions
## Base Exceptions
class HTTPError(Exception):
"Base exception used by this module."
pass
class SSLError(Exception):
class PoolError(HTTPError):
"Base exception for errors caused within a pool."
def __init__(self, pool, message):
self.pool = pool
HTTPError.__init__(self, "%s: %s" % (pool, message))
class SSLError(HTTPError):
"Raised when SSL certificate fails in an HTTPS connection."
pass
class MaxRetryError(HTTPError):
## Leaf Exceptions
class MaxRetryError(PoolError):
"Raised when the maximum number of retries is exceeded."
pass
def __init__(self, pool, url):
PoolError.__init__(self, pool,
"Max retries exceeded with url: %s" % url)
self.url = url
class TimeoutError(HTTPError):
class HostChangedError(PoolError):
"Raised when an existing pool gets a request for a foreign host."
def __init__(self, pool, url, retries=3):
PoolError.__init__(self, pool,
"Tried to open a foreign host with url: %s" % url)
self.url = url
self.retries = retries
class TimeoutError(PoolError):
"Raised when a socket timeout occurs."
pass
class HostChangedError(HTTPError):
"Raised when an existing pool gets a request for a foreign host."
pass
class EmptyPoolError(HTTPError):
class EmptyPoolError(PoolError):
"Raised when a pool runs out of connections and no more are allowed."
pass
libs/requests/packages/urllib3/filepost.py
+18 -15
View File
@@ -1,18 +1,21 @@
# urllib3/filepost.py
# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
# Copyright 2008-2012 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
#
# This module is part of urllib3 and is released under
# the MIT License: http://www.opensource.org/licenses/mit-license.php
import codecs
import mimetools
import mimetypes
try:
from cStringIO import StringIO
from mimetools import choose_boundary
except ImportError:
from StringIO import StringIO # pylint: disable-msg=W0404
from .packages.mimetools_choose_boundary import choose_boundary
from io import BytesIO
from .packages import six
from .packages.six import b
writer = codecs.lookup('utf-8')[3]
@@ -35,37 +38,37 @@ def encode_multipart_formdata(fields, boundary=None):
If not specified, then a random boundary will be generated using
:func:`mimetools.choose_boundary`.
"""
body = StringIO()
body = BytesIO()
if boundary is None:
boundary = mimetools.choose_boundary()
boundary = choose_boundary()
for fieldname, value in fields.iteritems():
body.write('--%s\r\n' % (boundary))
for fieldname, value in six.iteritems(fields):
body.write(b('--%s\r\n' % (boundary)))
if isinstance(value, tuple):
filename, data = value
writer(body).write('Content-Disposition: form-data; name="%s"; '
'filename="%s"\r\n' % (fieldname, filename))
body.write('Content-Type: %s\r\n\r\n' %
(get_content_type(filename)))
body.write(b('Content-Type: %s\r\n\r\n' %
(get_content_type(filename))))
else:
data = value
writer(body).write('Content-Disposition: form-data; name="%s"\r\n'
% (fieldname))
body.write('Content-Type: text/plain\r\n\r\n')
body.write(b'Content-Type: text/plain\r\n\r\n')
if isinstance(data, int):
data = str(data) # Backwards compatibility
if isinstance(data, unicode):
if isinstance(data, six.text_type):
writer(body).write(data)
else:
body.write(data)
body.write('\r\n')
body.write(b'\r\n')
body.write('--%s--\r\n' % (boundary))
body.write(b('--%s--\r\n' % (boundary)))
content_type = 'multipart/form-data; boundary=%s' % boundary
content_type = b('multipart/form-data; boundary=%s' % boundary)
return body.getvalue(), content_type
libs/requests/packages/urllib3/packages/mimetools_choose_boundary/__init__.py
+47
View File
@@ -0,0 +1,47 @@
"""The function mimetools.choose_boundary() from Python 2.7, which seems to
have disappeared in Python 3 (although email.generator._make_boundary() might
work as a replacement?).
Tweaked to use lock from threading rather than thread.
"""
import os
from threading import Lock
_counter_lock = Lock()
_counter = 0
def _get_next_counter():
global _counter
with _counter_lock:
_counter += 1
return _counter
_prefix = None
def choose_boundary():
"""Return a string usable as a multipart boundary.
The string chosen is unique within a single program run, and
incorporates the user id (if available), process id (if available),
and current time. So it's very unlikely the returned string appears
in message text, but there's no guarantee.
The boundary contains dots so you have to quote it in the header."""
global _prefix
import time
if _prefix is None:
import socket
try:
hostid = socket.gethostbyname(socket.gethostname())
except socket.gaierror:
hostid = '127.0.0.1'
try:
uid = repr(os.getuid())
except AttributeError:
uid = '1'
try:
pid = repr(os.getpid())
except AttributeError:
pid = '1'
_prefix = hostid + '.' + uid + '.' + pid
return "%s.%.3f.%d" % (_prefix, time.time(), _get_next_counter())
libs/requests/packages/urllib3/packages/six.py
+372
View File
@@ -0,0 +1,372 @@
"""Utilities for writing code that runs on Python 2 and 3"""
#Copyright (c) 2010-2011 Benjamin Peterson
#Permission is hereby granted, free of charge, to any person obtaining a copy of
#this software and associated documentation files (the "Software"), to deal in
#the Software without restriction, including without limitation the rights to
#use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
#the Software, and to permit persons to whom the Software is furnished to do so,
#subject to the following conditions:
#The above copyright notice and this permission notice shall be included in all
#copies or substantial portions of the Software.
#THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
#IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
#FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
#COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
#IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
#CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import operator
import sys
import types
__author__ = "Benjamin Peterson <benjamin@python.org>"
__version__ = "1.1.0"
# True if we are running on Python 3.
PY3 = sys.version_info[0] == 3
if PY3:
string_types = str,
integer_types = int,
class_types = type,
text_type = str
binary_type = bytes
MAXSIZE = sys.maxsize
else:
string_types = basestring,
integer_types = (int, long)
class_types = (type, types.ClassType)
text_type = unicode
binary_type = str
# It's possible to have sizeof(long) != sizeof(Py_ssize_t).
class X(object):
def __len__(self):
return 1 << 31
try:
len(X())
except OverflowError:
# 32-bit
MAXSIZE = int((1 << 31) - 1)
else:
# 64-bit
MAXSIZE = int((1 << 63) - 1)
del X
def _add_doc(func, doc):
"""Add documentation to a function."""
func.__doc__ = doc
def _import_module(name):
"""Import module, returning the module after the last dot."""
__import__(name)
return sys.modules[name]
class _LazyDescr(object):
def __init__(self, name):
self.name = name
def __get__(self, obj, tp):
result = self._resolve()
setattr(obj, self.name, result)
# This is a bit ugly, but it avoids running this again.
delattr(tp, self.name)
return result
class MovedModule(_LazyDescr):
def __init__(self, name, old, new=None):
super(MovedModule, self).__init__(name)
if PY3:
if new is None:
new = name
self.mod = new
else:
self.mod = old
def _resolve(self):
return _import_module(self.mod)
class MovedAttribute(_LazyDescr):
def __init__(self, name, old_mod, new_mod, old_attr=None, new_attr=None):
super(MovedAttribute, self).__init__(name)
if PY3:
if new_mod is None:
new_mod = name
self.mod = new_mod
if new_attr is None:
if old_attr is None:
new_attr = name
else:
new_attr = old_attr
self.attr = new_attr
else:
self.mod = old_mod
if old_attr is None:
old_attr = name
self.attr = old_attr
def _resolve(self):
module = _import_module(self.mod)
return getattr(module, self.attr)
class _MovedItems(types.ModuleType):
"""Lazy loading of moved objects"""
_moved_attributes = [
MovedAttribute("cStringIO", "cStringIO", "io", "StringIO"),
MovedAttribute("filter", "itertools", "builtins", "ifilter", "filter"),
MovedAttribute("map", "itertools", "builtins", "imap", "map"),
MovedAttribute("reload_module", "__builtin__", "imp", "reload"),
MovedAttribute("reduce", "__builtin__", "functools"),
MovedAttribute("StringIO", "StringIO", "io"),
MovedAttribute("xrange", "__builtin__", "builtins", "xrange", "range"),
MovedAttribute("zip", "itertools", "builtins", "izip", "zip"),
MovedModule("builtins", "__builtin__"),
MovedModule("configparser", "ConfigParser"),
MovedModule("copyreg", "copy_reg"),
MovedModule("http_cookiejar", "cookielib", "http.cookiejar"),
MovedModule("http_cookies", "Cookie", "http.cookies"),
MovedModule("html_entities", "htmlentitydefs", "html.entities"),
MovedModule("html_parser", "HTMLParser", "html.parser"),
MovedModule("http_client", "httplib", "http.client"),
MovedModule("BaseHTTPServer", "BaseHTTPServer", "http.server"),
MovedModule("CGIHTTPServer", "CGIHTTPServer", "http.server"),
MovedModule("SimpleHTTPServer", "SimpleHTTPServer", "http.server"),
MovedModule("cPickle", "cPickle", "pickle"),
MovedModule("queue", "Queue"),
MovedModule("reprlib", "repr"),
MovedModule("socketserver", "SocketServer"),
MovedModule("tkinter", "Tkinter"),
MovedModule("tkinter_dialog", "Dialog", "tkinter.dialog"),
MovedModule("tkinter_filedialog", "FileDialog", "tkinter.filedialog"),
MovedModule("tkinter_scrolledtext", "ScrolledText", "tkinter.scrolledtext"),
MovedModule("tkinter_simpledialog", "SimpleDialog", "tkinter.simpledialog"),
MovedModule("tkinter_tix", "Tix", "tkinter.tix"),
MovedModule("tkinter_constants", "Tkconstants", "tkinter.constants"),
MovedModule("tkinter_dnd", "Tkdnd", "tkinter.dnd"),
MovedModule("tkinter_colorchooser", "tkColorChooser",
"tkinter.colorchooser"),
MovedModule("tkinter_commondialog", "tkCommonDialog",
"tkinter.commondialog"),
MovedModule("tkinter_tkfiledialog", "tkFileDialog", "tkinter.filedialog"),
MovedModule("tkinter_font", "tkFont", "tkinter.font"),
MovedModule("tkinter_messagebox", "tkMessageBox", "tkinter.messagebox"),
MovedModule("tkinter_tksimpledialog", "tkSimpleDialog",
"tkinter.simpledialog"),
MovedModule("urllib_robotparser", "robotparser", "urllib.robotparser"),
MovedModule("winreg", "_winreg"),
]
for attr in _moved_attributes:
setattr(_MovedItems, attr.name, attr)
del attr
moves = sys.modules["six.moves"] = _MovedItems("moves")
def add_move(move):
"""Add an item to six.moves."""
setattr(_MovedItems, move.name, move)
def remove_move(name):
"""Remove item from six.moves."""
try:
delattr(_MovedItems, name)
except AttributeError:
try:
del moves.__dict__[name]
except KeyError:
raise AttributeError("no such move, %r" % (name,))
if PY3:
_meth_func = "__func__"
_meth_self = "__self__"
_func_code = "__code__"
_func_defaults = "__defaults__"
_iterkeys = "keys"
_itervalues = "values"
_iteritems = "items"
else:
_meth_func = "im_func"
_meth_self = "im_self"
_func_code = "func_code"
_func_defaults = "func_defaults"
_iterkeys = "iterkeys"
_itervalues = "itervalues"
_iteritems = "iteritems"
if PY3:
def get_unbound_function(unbound):
return unbound
advance_iterator = next
def callable(obj):
return any("__call__" in klass.__dict__ for klass in type(obj).__mro__)
else:
def get_unbound_function(unbound):
return unbound.im_func
def advance_iterator(it):
return it.next()
callable = callable
_add_doc(get_unbound_function,
"""Get the function out of a possibly unbound function""")
get_method_function = operator.attrgetter(_meth_func)
get_method_self = operator.attrgetter(_meth_self)
get_function_code = operator.attrgetter(_func_code)
get_function_defaults = operator.attrgetter(_func_defaults)
def iterkeys(d):
"""Return an iterator over the keys of a dictionary."""
return getattr(d, _iterkeys)()
def itervalues(d):
"""Return an iterator over the values of a dictionary."""
return getattr(d, _itervalues)()
def iteritems(d):
"""Return an iterator over the (key, value) pairs of a dictionary."""
return getattr(d, _iteritems)()
if PY3:
def b(s):
return s.encode("latin-1")
def u(s):
return s
if sys.version_info[1] <= 1:
def int2byte(i):
return bytes((i,))
else:
# This is about 2x faster than the implementation above on 3.2+
int2byte = operator.methodcaller("to_bytes", 1, "big")
import io
StringIO = io.StringIO
BytesIO = io.BytesIO
else:
def b(s):
return s
def u(s):
return unicode(s, "unicode_escape")
int2byte = chr
import StringIO
StringIO = BytesIO = StringIO.StringIO
_add_doc(b, """Byte literal""")
_add_doc(u, """Text literal""")
if PY3:
import builtins
exec_ = getattr(builtins, "exec")
def reraise(tp, value, tb=None):
if value.__traceback__ is not tb:
raise value.with_traceback(tb)
raise value
print_ = getattr(builtins, "print")
del builtins
else:
def exec_(code, globs=None, locs=None):
"""Execute code in a namespace."""
if globs is None:
frame = sys._getframe(1)
globs = frame.f_globals
if locs is None:
locs = frame.f_locals
del frame
elif locs is None:
locs = globs
exec("""exec code in globs, locs""")
exec_("""def reraise(tp, value, tb=None):
raise tp, value, tb
""")
def print_(*args, **kwargs):
"""The new-style print function."""
fp = kwargs.pop("file", sys.stdout)
if fp is None:
return
def write(data):
if not isinstance(data, basestring):
data = str(data)
fp.write(data)
want_unicode = False
sep = kwargs.pop("sep", None)
if sep is not None:
if isinstance(sep, unicode):
want_unicode = True
elif not isinstance(sep, str):
raise TypeError("sep must be None or a string")
end = kwargs.pop("end", None)
if end is not None:
if isinstance(end, unicode):
want_unicode = True
elif not isinstance(end, str):
raise TypeError("end must be None or a string")
if kwargs:
raise TypeError("invalid keyword arguments to print()")
if not want_unicode:
for arg in args:
if isinstance(arg, unicode):
want_unicode = True
break
if want_unicode:
newline = unicode("\n")
space = unicode(" ")
else:
newline = "\n"
space = " "
if sep is None:
sep = space
if end is None:
end = newline
for i, arg in enumerate(args):
if i:
write(sep)
write(arg)
write(end)
_add_doc(reraise, """Reraise an exception.""")
def with_metaclass(meta, base=object):
"""Create a base class with a metaclass."""
return meta("NewBase", (base,), {})
libs/requests/packages/urllib3/poolmanager.py
+14 -14
View File
@@ -1,32 +1,27 @@
# urllib3/poolmanager.py
# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
# Copyright 2008-2012 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
#
# This module is part of urllib3 and is released under
# the MIT License: http://www.opensource.org/licenses/mit-license.php
import logging
from ._collections import RecentlyUsedContainer
from .connectionpool import (
HTTPConnectionPool, HTTPSConnectionPool,
get_host, connection_from_url,
)
from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool
from .connectionpool import get_host, connection_from_url, port_by_scheme
from .exceptions import HostChangedError
from .request import RequestMethods
__all__ = ['PoolManager', 'ProxyManager', 'proxy_from_url']
from .request import RequestMethods
from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool
pool_classes_by_scheme = {
'http': HTTPConnectionPool,
'https': HTTPSConnectionPool,
}
port_by_scheme = {
'http': 80,
'https': 443,
}
log = logging.getLogger(__name__)
class PoolManager(RequestMethods):
@@ -105,7 +100,12 @@ class PoolManager(RequestMethods):
:class:`urllib3.connectionpool.ConnectionPool` can be chosen for it.
"""
conn = self.connection_from_url(url)
return conn.urlopen(method, url, assert_same_host=False, **kw)
try:
return conn.urlopen(method, url, **kw)
except HostChangedError as e:
kw['retries'] = e.retries # Persist retries countdown
return self.urlopen(method, e.url, **kw)
class ProxyManager(RequestMethods):
libs/requests/packages/urllib3/request.py
+5 -3
View File
@@ -1,11 +1,13 @@
# urllib3/request.py
# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
# Copyright 2008-2012 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
#
# This module is part of urllib3 and is released under
# the MIT License: http://www.opensource.org/licenses/mit-license.php
from urllib import urlencode
try:
from urllib.parse import urlencode
except ImportError:
from urllib import urlencode
from .filepost import encode_multipart_formdata
libs/requests/packages/urllib3/response.py
+50 -35
View File
@@ -1,5 +1,5 @@
# urllib3/response.py
# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
# Copyright 2008-2012 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
#
# This module is part of urllib3 and is released under
# the MIT License: http://www.opensource.org/licenses/mit-license.php
@@ -8,21 +8,22 @@ import gzip
import logging
import zlib
try:
from cStringIO import StringIO
except ImportError:
from StringIO import StringIO # pylint: disable-msg=W0404
from io import BytesIO
from .exceptions import HTTPError
try:
basestring = basestring
except NameError: # Python 3
basestring = (str, bytes)
log = logging.getLogger(__name__)
def decode_gzip(data):
gzipper = gzip.GzipFile(fileobj=StringIO(data))
gzipper = gzip.GzipFile(fileobj=BytesIO(data))
return gzipper.read()
@@ -71,7 +72,7 @@ class HTTPResponse(object):
self.strict = strict
self._decode_content = decode_content
self._body = None
self._body = body if body and isinstance(body, basestring) else None
self._fp = None
self._original_response = original_response
@@ -81,9 +82,22 @@ class HTTPResponse(object):
if hasattr(body, 'read'):
self._fp = body
if preload_content:
if preload_content and not self._body:
self._body = self.read(decode_content=decode_content)
def get_redirect_location(self):
"""
Should we redirect and where to?
:returns: Truthy redirect location string if we got a redirect status
code and valid location. ``None`` if redirect status and no
location. ``False`` if not a redirect status code.
"""
if self.status in [301, 302, 303, 307]:
return self.headers.get('location')
return False
def release_conn(self):
if not self._pool or not self._connection:
return
@@ -98,10 +112,9 @@ class HTTPResponse(object):
return self._body
if self._fp:
return self.read(decode_content=self._decode_content,
cache_content=True)
return self.read(cache_content=True)
def read(self, amt=None, decode_content=True, cache_content=False):
def read(self, amt=None, decode_content=None, cache_content=False):
"""
Similar to :meth:`httplib.HTTPResponse.read`, but with two additional
parameters: ``decode_content`` and ``cache_content``.
@@ -124,22 +137,22 @@ class HTTPResponse(object):
"""
content_encoding = self.headers.get('content-encoding')
decoder = self.CONTENT_DECODERS.get(content_encoding)
if decode_content is None:
decode_content = self._decode_content
data = self._fp and self._fp.read(amt)
if self._fp is None:
return
try:
if amt:
return data
if not decode_content or not decoder:
if cache_content:
self._body = data
return data
if amt is None:
# cStringIO doesn't like amt=None
data = self._fp.read()
else:
return self._fp.read(amt)
try:
data = decoder(data)
if decode_content and decoder:
data = decoder(data)
except IOError:
raise HTTPError("Received response with content-encoding: %s, but "
"failed to decode it." % content_encoding)
@@ -150,12 +163,11 @@ class HTTPResponse(object):
return data
finally:
if self._original_response and self._original_response.isclosed():
self.release_conn()
@staticmethod
def from_httplib(r, **response_kw):
@classmethod
def from_httplib(ResponseCls, r, **response_kw):
"""
Given an :class:`httplib.HTTPResponse` instance ``r``, return a
corresponding :class:`urllib3.response.HTTPResponse` object.
@@ -164,14 +176,17 @@ class HTTPResponse(object):
with ``original_response=r``.
"""
return HTTPResponse(body=r,
headers=dict(r.getheaders()),
status=r.status,
version=r.version,
reason=r.reason,
strict=r.strict,
original_response=r,
**response_kw)
# HTTPResponse objects in Python 3 don't have a .strict attribute
strict = getattr(r, 'strict', 0)
return ResponseCls(body=r,
# In Python 3, the header keys are returned capitalised
headers=dict((k.lower(), v) for k,v in r.getheaders()),
status=r.status,
version=r.version,
reason=r.reason,
strict=strict,
original_response=r,
**response_kw)
# Backwards-compatibility methods for httplib.HTTPResponse
def getheaders(self):
libs/requests/sessions.py
+5 -5
View File
@@ -25,7 +25,7 @@ def merge_kwargs(local_kwarg, default_kwarg):
if default_kwarg is None:
return local_kwarg
if isinstance(local_kwarg, basestring):
if isinstance(local_kwarg, str):
return local_kwarg
if local_kwarg is None:
@@ -40,7 +40,7 @@ def merge_kwargs(local_kwarg, default_kwarg):
kwargs.update(local_kwarg)
# Remove keys that are set to None.
for (k,v) in local_kwarg.items():
for (k,v) in list(local_kwarg.items()):
if v is None:
del kwargs[k]
@@ -76,7 +76,7 @@ class Session(object):
self.config = config or {}
self.verify = verify
for (k, v) in defaults.items():
for (k, v) in list(defaults.items()):
self.config.setdefault(k, v)
self.poolmanager = PoolManager(
@@ -150,12 +150,12 @@ class Session(object):
verify = self.verify
# use session's hooks as defaults
for key, cb in self.hooks.iteritems():
for key, cb in list(self.hooks.items()):
hooks.setdefault(key, cb)
# Expand header values.
if headers:
for k, v in headers.items() or {}:
for k, v in list(headers.items()) or {}:
headers[k] = header_expand(v)
args = dict(
libs/requests/status_codes.py
+1 -1
View File
@@ -79,7 +79,7 @@ _codes = {
codes = LookupDict(name='status_codes')
for (code, titles) in _codes.items():
for (code, titles) in list(_codes.items()):
for title in titles:
setattr(codes, title, code)
if not title.startswith('\\'):
libs/requests/structures.py
+2 -2
View File
@@ -18,7 +18,7 @@ class CaseInsensitiveDict(dict):
@property
def lower_keys(self):
if not hasattr(self, '_lower_keys') or not self._lower_keys:
self._lower_keys = dict((k.lower(), k) for k in self.iterkeys())
self._lower_keys = dict((k.lower(), k) for k in list(self.keys()))
return self._lower_keys
def _clear_lower_keys(self):
@@ -63,4 +63,4 @@ class LookupDict(dict):
return self.__dict__.get(key, None)
def get(self, key, default=None):
return self.__dict__.get(key, default)
return self.__dict__.get(key, default)
libs/requests/utils.py
+35 -26
View File
@@ -11,16 +11,28 @@ that are also useful for external consumption.
import cgi
import codecs
import cookielib
import os
import random
import re
import zlib
import urllib
from urllib2 import parse_http_list as _parse_list_header
from .compat import parse_http_list as _parse_list_header
from .compat import quote, unquote, cookielib, SimpleCookie, is_py2
def dict_from_string(s):
"""Returns a MultiDict with Cookies."""
cookies = dict()
c = SimpleCookie()
c.load(s)
for k,v in list(c.items()):
cookies.update({k: v.value})
return cookies
def guess_filename(obj):
"""Tries to guess the filename of the given object."""
name = getattr(obj, 'name', None)
@@ -132,16 +144,16 @@ def header_expand(headers):
collector = []
if isinstance(headers, dict):
headers = headers.items()
headers = list(headers.items())
elif isinstance(headers, basestring):
elif isinstance(headers, str):
return headers
for i, (value, params) in enumerate(headers):
_params = []
for (p_k, p_v) in params.items():
for (p_k, p_v) in list(params.items()):
_params.append('%s=%s' % (p_k, p_v))
@@ -166,17 +178,11 @@ def header_expand(headers):
def randombytes(n):
"""Return n random bytes."""
# Use /dev/urandom if it is available. Fall back to random module
# if not. It might be worthwhile to extend this function to use
# other platform-specific mechanisms for getting random bytes.
if os.path.exists("/dev/urandom"):
f = open("/dev/urandom")
s = f.read(n)
f.close()
return s
else:
if is_py2:
L = [chr(random.randrange(0, 256)) for i in range(n)]
return "".join(L)
else:
L = [chr(random.randrange(0, 256)).encode('utf-8') for i in range(n)]
return b"".join(L)
def dict_from_cookiejar(cj):
@@ -187,9 +193,9 @@ def dict_from_cookiejar(cj):
cookie_dict = {}
for _, cookies in cj._cookies.items():
for _, cookies in cookies.items():
for cookie in cookies.values():
for _, cookies in list(cj._cookies.items()):
for _, cookies in list(cookies.items()):
for cookie in list(cookies.values()):
# print cookie
cookie_dict[cookie.name] = cookie.value
@@ -221,7 +227,7 @@ def add_dict_to_cookiejar(cj, cookie_dict):
:param cookie_dict: Dict of key/values to insert into CookieJar.
"""
for k, v in cookie_dict.items():
for k, v in list(cookie_dict.items()):
cookie = cookielib.Cookie(
version=0,
@@ -276,6 +282,9 @@ def get_encoding_from_headers(headers):
if 'charset' in params:
return params['charset'].strip("'\"")
if 'text' in content_type:
return 'ISO-8859-1'
def unicode_from_html(content):
"""Attempts to decode an HTML string into unicode.
@@ -287,7 +296,7 @@ def unicode_from_html(content):
for encoding in encodings:
try:
return unicode(content, encoding)
return str(content, encoding)
except (UnicodeError, TypeError):
pass
@@ -334,13 +343,13 @@ def get_unicode_from_response(r):
if encoding:
try:
return unicode(r.content, encoding)
return str(r.content, encoding)
except UnicodeError:
tried_encodings.append(encoding)
# Fall back:
try:
return unicode(r.content, encoding, errors='replace')
return str(r.content, encoding, errors='replace')
except TypeError:
return r.content
@@ -393,6 +402,6 @@ def requote_path(path):
This function passes the given path through an unquote/quote cycle to
ensure that it is fully and consistently quoted.
"""
parts = path.split("/")
parts = (urllib.quote(urllib.unquote(part), safe="") for part in parts)
return "/".join(parts)
parts = path.split(b"/")
parts = (quote(unquote(part), safe=b"") for part in parts)
return b"/".join(parts)