dinislage/CouchPotatoServer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update libs

Browse Source
This commit is contained in:
Antoine Bertin
2012-06-24 23:28:38 +02:00
parent 2d05aafc04
commit 88b71c8116
157 changed files with 15686 additions and 2017 deletions
Download Patch File Download Diff File Expand all files Collapse all files
libs/requests/__init__.py
+2 -2
View File
@@ -15,8 +15,8 @@ requests
"""
__title__ = 'requests'
__version__ = '0.11.1'
__build__ = 0x001101
__version__ = '0.13.1'
__build__ = 0x001301
__author__ = 'Kenneth Reitz'
__license__ = 'ISC'
__copyright__ = 'Copyright 2012 Kenneth Reitz'
libs/requests/api.py
+3 -1
View File
@@ -12,7 +12,9 @@ This module implements the Requests API.
"""
from . import sessions
from .safe_mode import catch_exceptions_if_in_safe_mode
@catch_exceptions_if_in_safe_mode
def request(method, url, **kwargs):
"""Constructs and sends a :class:`Request <Request>`.
Returns :class:`Response <Response>` object.
@@ -30,7 +32,7 @@ def request(method, url, **kwargs):
:param proxies: (optional) Dictionary mapping protocol to the URL of the proxy.
:param return_response: (optional) If False, an un-sent Request object will returned.
:param session: (optional) A :class:`Session` object to be used for the request.
:param config: (optional) A configuration dictionary.
:param config: (optional) A configuration dictionary. See ``request.defaults`` for allowed keys and their default values.
:param verify: (optional) if ``True``, the SSL cert will be verified. A CA_BUNDLE path can also be provided.
:param prefetch: (optional) if ``True``, the response content will be immediately downloaded.
:param cert: (optional) if String, path to ssl client cert file (.pem). If Tuple, ('cert', 'key') pair.
libs/requests/async.py
-106
View File
@@ -1,106 +0,0 @@
# -*- coding: utf-8 -*-
"""
requests.async
~~~~~~~~~~~~~~
This module contains an asynchronous replica of ``requests.api``, powered
by gevent. All API methods return a ``Request`` instance (as opposed to
``Response``). A list of requests can be sent with ``map()``.
"""
try:
import gevent
from gevent import monkey as curious_george
from gevent.pool import Pool
except ImportError:
raise RuntimeError('Gevent is required for requests.async.')
# Monkey-patch.
curious_george.patch_all(thread=False, select=False)
from . import api
__all__ = (
'map', 'imap',
'get', 'options', 'head', 'post', 'put', 'patch', 'delete', 'request'
)
def patched(f):
"""Patches a given API function to not send."""
def wrapped(*args, **kwargs):
kwargs['return_response'] = False
kwargs['prefetch'] = True
config = kwargs.get('config', {})
config.update(safe_mode=True)
kwargs['config'] = config
return f(*args, **kwargs)
return wrapped
def send(r, pool=None, prefetch=False):
"""Sends the request object using the specified pool. If a pool isn't
specified this method blocks. Pools are useful because you can specify size
and can hence limit concurrency."""
if pool != None:
return pool.spawn(r.send, prefetch=prefetch)
return gevent.spawn(r.send, prefetch=prefetch)
# Patched requests.api functions.
get = patched(api.get)
options = patched(api.options)
head = patched(api.head)
post = patched(api.post)
put = patched(api.put)
patch = patched(api.patch)
delete = patched(api.delete)
request = patched(api.request)
def map(requests, prefetch=True, size=None):
"""Concurrently converts a list of Requests to Responses.
:param requests: a collection of Request objects.
:param prefetch: If False, the content will not be downloaded immediately.
:param size: Specifies the number of requests to make at a time. If None, no throttling occurs.
"""
requests = list(requests)
pool = Pool(size) if size else None
jobs = [send(r, pool, prefetch=prefetch) for r in requests]
gevent.joinall(jobs)
return [r.response for r in requests]
def imap(requests, prefetch=True, size=2):
"""Concurrently converts a generator object of Requests to
a generator of Responses.
:param requests: a generator of Request objects.
:param prefetch: If False, the content will not be downloaded immediately.
:param size: Specifies the number of requests to make at a time. default is 2
"""
pool = Pool(size)
def send(r):
r.send(prefetch)
return r.response
for r in pool.imap_unordered(send, requests):
yield r
pool.join()
libs/requests/auth.py
+92 -11
View File
@@ -7,14 +7,25 @@ requests.auth
This module contains the authentication handlers for Requests.
"""
import os
import time
import hashlib
from base64 import b64encode
from .compat import urlparse, str
from .utils import randombytes, parse_dict_header
from .utils import parse_dict_header
try:
from oauthlib.oauth1.rfc5849 import (Client, SIGNATURE_HMAC, SIGNATURE_TYPE_AUTH_HEADER)
from oauthlib.common import extract_params
# hush pyflakes:
SIGNATURE_HMAC; SIGNATURE_TYPE_AUTH_HEADER
except (ImportError, SyntaxError):
SIGNATURE_HMAC = None
SIGNATURE_TYPE_AUTH_HEADER = None
CONTENT_TYPE_FORM_URLENCODED = 'application/x-www-form-urlencoded'
def _basic_auth_str(username, password):
"""Returns a Basic Auth string."""
@@ -29,6 +40,74 @@ class AuthBase(object):
raise NotImplementedError('Auth hooks must be callable.')
class OAuth1(AuthBase):
"""Signs the request using OAuth 1 (RFC5849)"""
def __init__(self, client_key,
client_secret=None,
resource_owner_key=None,
resource_owner_secret=None,
callback_uri=None,
signature_method=SIGNATURE_HMAC,
signature_type=SIGNATURE_TYPE_AUTH_HEADER,
rsa_key=None, verifier=None):
try:
signature_type = signature_type.upper()
except AttributeError:
pass
self.client = Client(client_key, client_secret, resource_owner_key,
resource_owner_secret, callback_uri, signature_method,
signature_type, rsa_key, verifier)
def __call__(self, r):
"""Add OAuth parameters to the request.
Parameters may be included from the body if the content-type is
urlencoded, if no content type is set an educated guess is made.
"""
contenttype = r.headers.get('Content-Type', None)
# extract_params will not give params unless the body is a properly
# formatted string, a dictionary or a list of 2-tuples.
decoded_body = extract_params(r.data)
if contenttype == None and decoded_body != None:
# extract_params can only check the present r.data and does not know
# of r.files, thus an extra check is performed. We know that
# if files are present the request will not have
# Content-type: x-www-form-urlencoded. We guess it will have
# a mimetype of multipart/form-encoded and if this is not the case
# we assume the correct header will be set later.
if r.files:
# Omit body data in the signing and since it will always
# be empty (cant add paras to body if multipart) and we wish
# to preserve body.
r.headers['Content-Type'] = 'multipart/form-encoded'
r.url, r.headers, _ = self.client.sign(
unicode(r.url), unicode(r.method), None, r.headers)
else:
# Normal signing
r.headers['Content-Type'] = 'application/x-www-form-urlencoded'
r.url, r.headers, r.data = self.client.sign(
unicode(r.url), unicode(r.method), r.data, r.headers)
# Having the authorization header, key or value, in unicode will
# result in UnicodeDecodeErrors when the request is concatenated
# by httplib. This can easily be seen when attaching files.
# Note that simply encoding the value is not enough since Python
# saves the type of first key set. Thus we remove and re-add.
# >>> d = {u'a':u'foo'}
# >>> d['a'] = 'foo'
# >>> d
# { u'a' : 'foo' }
u_header = unicode('Authorization')
if u_header in r.headers:
auth_header = r.headers[u_header].encode('utf-8')
del r.headers[u_header]
r.headers['Authorization'] = auth_header
return r
class HTTPBasicAuth(AuthBase):
"""Attaches HTTP Basic Authentication to the given Request object."""
def __init__(self, username, password):
@@ -56,6 +135,8 @@ class HTTPDigestAuth(AuthBase):
def handle_401(self, r):
"""Takes the given response and tries digest-auth, if needed."""
r.request.deregister_hook('response', self.handle_401)
s_auth = r.headers.get('www-authenticate', '')
if 'digest' in s_auth.lower():
@@ -74,21 +155,21 @@ class HTTPDigestAuth(AuthBase):
algorithm = algorithm.upper()
# lambdas assume digest modules are imported at the top level
if algorithm == 'MD5':
def h(x):
def md5_utf8(x):
if isinstance(x, str):
x = x.encode('utf-8')
return hashlib.md5(x).hexdigest()
H = h
hash_utf8 = md5_utf8
elif algorithm == 'SHA':
def h(x):
def sha_utf8(x):
if isinstance(x, str):
x = x.encode('utf-8')
return hashlib.sha1(x).hexdigest()
H = h
hash_utf8 = sha_utf8
# XXX MD5-sess
KD = lambda s, d: H("%s:%s" % (s, d))
KD = lambda s, d: hash_utf8("%s:%s" % (s, d))
if H is None:
if hash_utf8 is None:
return None
# XXX not implemented yet
@@ -112,13 +193,13 @@ class HTTPDigestAuth(AuthBase):
s = str(nonce_count).encode('utf-8')
s += nonce.encode('utf-8')
s += time.ctime().encode('utf-8')
s += randombytes(8)
s += os.urandom(8)
cnonce = (hashlib.sha1(s).hexdigest()[:16])
noncebit = "%s:%s:%s:%s:%s" % (nonce, ncvalue, cnonce, qop, H(A2))
respdig = KD(H(A1), noncebit)
noncebit = "%s:%s:%s:%s:%s" % (nonce, ncvalue, cnonce, qop, hash_utf8(A2))
respdig = KD(hash_utf8(A1), noncebit)
elif qop is None:
respdig = KD(H(A1), "%s:%s" % (nonce, H(A2)))
respdig = KD(hash_utf8(A1), "%s:%s" % (nonce, hash_utf8(A2)))
else:
# XXX handle auth-int.
return None
libs/requests/compat.py
+2 -2
View File
@@ -83,7 +83,7 @@ if is_py2:
from urlparse import urlparse, urlunparse, urljoin, urlsplit
from urllib2 import parse_http_list
import cookielib
from .packages.oreos.monkeys import SimpleCookie
from Cookie import Morsel
from StringIO import StringIO
bytes = str
@@ -96,7 +96,7 @@ elif is_py3:
from urllib.parse import urlparse, urlunparse, urljoin, urlsplit, urlencode, quote, unquote
from urllib.request import parse_http_list
from http import cookiejar as cookielib
from http.cookies import SimpleCookie
from http.cookies import Morsel
from io import StringIO
str = str
libs/requests/cookies.py
+363
View File
@@ -0,0 +1,363 @@
"""
Compatibility code to be able to use `cookielib.CookieJar` with requests.
requests.utils imports from here, so be careful with imports.
"""
import collections
from .compat import cookielib, urlparse, Morsel
try:
import threading
# grr, pyflakes: this fixes "redefinition of unused 'threading'"
threading
except ImportError:
import dummy_threading as threading
class MockRequest(object):
"""Wraps a `requests.Request` to mimic a `urllib2.Request`.
The code in `cookielib.CookieJar` expects this interface in order to correctly
manage cookie policies, i.e., determine whether a cookie can be set, given the
domains of the request and the cookie.
The original request object is read-only. The client is responsible for collecting
the new headers via `get_new_headers()` and interpreting them appropriately. You
probably want `get_cookie_header`, defined below.
"""
def __init__(self, request):
self._r = request
self._new_headers = {}
def get_type(self):
return urlparse(self._r.full_url).scheme
def get_host(self):
return urlparse(self._r.full_url).netloc
def get_origin_req_host(self):
if self._r.response.history:
r = self._r.response.history[0]
return urlparse(r).netloc
else:
return self.get_host()
def get_full_url(self):
return self._r.full_url
def is_unverifiable(self):
# unverifiable == redirected
return bool(self._r.response.history)
def has_header(self, name):
return name in self._r.headers or name in self._new_headers
def get_header(self, name, default=None):
return self._r.headers.get(name, self._new_headers.get(name, default))
def add_header(self, key, val):
"""cookielib has no legitimate use for this method; add it back if you find one."""
raise NotImplementedError("Cookie headers should be added with add_unredirected_header()")
def add_unredirected_header(self, name, value):
self._new_headers[name] = value
def get_new_headers(self):
return self._new_headers
class MockResponse(object):
"""Wraps a `httplib.HTTPMessage` to mimic a `urllib.addinfourl`.
...what? Basically, expose the parsed HTTP headers from the server response
the way `cookielib` expects to see them.
"""
def __init__(self, headers):
"""Make a MockResponse for `cookielib` to read.
:param headers: a httplib.HTTPMessage or analogous carrying the headers
"""
self._headers = headers
def info(self):
return self._headers
def getheaders(self, name):
self._headers.getheaders(name)
def extract_cookies_to_jar(jar, request, response):
"""Extract the cookies from the response into a CookieJar.
:param jar: cookielib.CookieJar (not necessarily a RequestsCookieJar)
:param request: our own requests.Request object
:param response: urllib3.HTTPResponse object
"""
# the _original_response field is the wrapped httplib.HTTPResponse object,
req = MockRequest(request)
# pull out the HTTPMessage with the headers and put it in the mock:
res = MockResponse(response._original_response.msg)
jar.extract_cookies(res, req)
def get_cookie_header(jar, request):
"""Produce an appropriate Cookie header string to be sent with `request`, or None."""
r = MockRequest(request)
jar.add_cookie_header(r)
return r.get_new_headers().get('Cookie')
def remove_cookie_by_name(cookiejar, name, domain=None, path=None):
"""Unsets a cookie by name, by default over all domains and paths.
Wraps CookieJar.clear(), is O(n).
"""
clearables = []
for cookie in cookiejar:
if cookie.name == name:
if domain is None or domain == cookie.domain:
if path is None or path == cookie.path:
clearables.append((cookie.domain, cookie.path, cookie.name))
for domain, path, name in clearables:
cookiejar.clear(domain, path, name)
class CookieConflictError(RuntimeError):
"""There are two cookies that meet the criteria specified in the cookie jar.
Use .get and .set and include domain and path args in order to be more specific."""
class RequestsCookieJar(cookielib.CookieJar, collections.MutableMapping):
"""Compatibility class; is a cookielib.CookieJar, but exposes a dict interface.
This is the CookieJar we create by default for requests and sessions that
don't specify one, since some clients may expect response.cookies and
session.cookies to support dict operations.
Don't use the dict interface internally; it's just for compatibility with
with external client code. All `requests` code should work out of the box
with externally provided instances of CookieJar, e.g., LWPCookieJar and
FileCookieJar.
Caution: dictionary operations that are normally O(1) may be O(n).
Unlike a regular CookieJar, this class is pickleable.
"""
def get(self, name, default=None, domain=None, path=None):
"""Dict-like get() that also supports optional domain and path args in
order to resolve naming collisions from using one cookie jar over
multiple domains. Caution: operation is O(n), not O(1)."""
try:
return self._find_no_duplicates(name, domain, path)
except KeyError:
return default
def set(self, name, value, **kwargs):
"""Dict-like set() that also supports optional domain and path args in
order to resolve naming collisions from using one cookie jar over
multiple domains."""
# support client code that unsets cookies by assignment of a None value:
if value is None:
remove_cookie_by_name(self, name, domain=kwargs.get('domain'), path=kwargs.get('path'))
return
if isinstance(value, Morsel):
c = morsel_to_cookie(value)
else:
c = create_cookie(name, value, **kwargs)
self.set_cookie(c)
return c
def keys(self):
"""Dict-like keys() that returns a list of names of cookies from the jar.
See values() and items()."""
keys = []
for cookie in iter(self):
keys.append(cookie.name)
return keys
def values(self):
"""Dict-like values() that returns a list of values of cookies from the jar.
See keys() and items()."""
values = []
for cookie in iter(self):
values.append(cookie.value)
return values
def items(self):
"""Dict-like items() that returns a list of name-value tuples from the jar.
See keys() and values(). Allows client-code to call "dict(RequestsCookieJar)
and get a vanilla python dict of key value pairs."""
items = []
for cookie in iter(self):
items.append((cookie.name, cookie.value))
return items
def list_domains(self):
"""Utility method to list all the domains in the jar."""
domains = []
for cookie in iter(self):
if cookie.domain not in domains:
domains.append(cookie.domain)
return domains
def list_paths(self):
"""Utility method to list all the paths in the jar."""
paths = []
for cookie in iter(self):
if cookie.path not in paths:
paths.append(cookie.path)
return paths
def multiple_domains(self):
"""Returns True if there are multiple domains in the jar.
Returns False otherwise."""
domains = []
for cookie in iter(self):
if cookie.domain is not None and cookie.domain in domains:
return True
domains.append(cookie.domain)
return False # there is only one domain in jar
def get_dict(self, domain=None, path=None):
"""Takes as an argument an optional domain and path and returns a plain old
Python dict of name-value pairs of cookies that meet the requirements."""
dictionary = {}
for cookie in iter(self):
if (domain == None or cookie.domain == domain) and (path == None
or cookie.path == path):
dictionary[cookie.name] = cookie.value
return dictionary
def __getitem__(self, name):
"""Dict-like __getitem__() for compatibility with client code. Throws exception
if there are more than one cookie with name. In that case, use the more
explicit get() method instead. Caution: operation is O(n), not O(1)."""
return self._find_no_duplicates(name)
def __setitem__(self, name, value):
"""Dict-like __setitem__ for compatibility with client code. Throws exception
if there is already a cookie of that name in the jar. In that case, use the more
explicit set() method instead."""
self.set(name, value)
def __delitem__(self, name):
"""Deletes a cookie given a name. Wraps cookielib.CookieJar's remove_cookie_by_name()."""
remove_cookie_by_name(self, name)
def _find(self, name, domain=None, path=None):
"""Requests uses this method internally to get cookie values. Takes as args name
and optional domain and path. Returns a cookie.value. If there are conflicting cookies,
_find arbitrarily chooses one. See _find_no_duplicates if you want an exception thrown
if there are conflicting cookies."""
for cookie in iter(self):
if cookie.name == name:
if domain is None or cookie.domain == domain:
if path is None or cookie.path == path:
return cookie.value
raise KeyError('name=%r, domain=%r, path=%r' % (name, domain, path))
def _find_no_duplicates(self, name, domain=None, path=None):
"""__get_item__ and get call _find_no_duplicates -- never used in Requests internally.
Takes as args name and optional domain and path. Returns a cookie.value.
Throws KeyError if cookie is not found and CookieConflictError if there are
multiple cookies that match name and optionally domain and path."""
toReturn = None
for cookie in iter(self):
if cookie.name == name:
if domain is None or cookie.domain == domain:
if path is None or cookie.path == path:
if toReturn != None: # if there are multiple cookies that meet passed in criteria
raise CookieConflictError('There are multiple cookies with name, %r' % (name))
toReturn = cookie.value # we will eventually return this as long as no cookie conflict
if toReturn:
return toReturn
raise KeyError('name=%r, domain=%r, path=%r' % (name, domain, path))
def __getstate__(self):
"""Unlike a normal CookieJar, this class is pickleable."""
state = self.__dict__.copy()
# remove the unpickleable RLock object
state.pop('_cookies_lock')
return state
def __setstate__(self, state):
"""Unlike a normal CookieJar, this class is pickleable."""
self.__dict__.update(state)
if '_cookies_lock' not in self.__dict__:
self._cookies_lock = threading.RLock()
def copy(self):
"""This is not implemented. Calling this will throw an exception."""
raise NotImplementedError
def create_cookie(name, value, **kwargs):
"""Make a cookie from underspecified parameters.
By default, the pair of `name` and `value` will be set for the domain ''
and sent on every request (this is sometimes called a "supercookie").
"""
result = dict(
version=0,
name=name,
value=value,
port=None,
domain='',
path='/',
secure=False,
expires=None,
discard=True,
comment=None,
comment_url=None,
rest={'HttpOnly': None},
rfc2109=False,
)
badargs = set(kwargs) - set(result)
if badargs:
err = 'create_cookie() got unexpected keyword arguments: %s'
raise TypeError(err % list(badargs))
result.update(kwargs)
result['port_specified'] = bool(result['port'])
result['domain_specified'] = bool(result['domain'])
result['domain_initial_dot'] = result['domain'].startswith('.')
result['path_specified'] = bool(result['path'])
return cookielib.Cookie(**result)
def morsel_to_cookie(morsel):
"""Convert a Morsel object into a Cookie containing the one k/v pair."""
c = create_cookie(
name=morsel.key,
value=morsel.value,
version=morsel['version'] or 0,
port=None,
port_specified=False,
domain=morsel['domain'],
domain_specified=bool(morsel['domain']),
domain_initial_dot=morsel['domain'].startswith('.'),
path=morsel['path'],
path_specified=bool(morsel['path']),
secure=bool(morsel['secure']),
expires=morsel['max-age'] or morsel['expires'],
discard=False,
comment=morsel['comment'],
comment_url=bool(morsel['comment']),
rest={'HttpOnly': morsel['httponly']},
rfc2109=False,
)
return c
def cookiejar_from_dict(cookie_dict, cookiejar=None):
"""Returns a CookieJar from a key/value dictionary.
:param cookie_dict: Dict of key/values to insert into CookieJar.
"""
if cookiejar is None:
cookiejar = RequestsCookieJar()
if cookie_dict is not None:
for name in cookie_dict:
cookiejar.set_cookie(create_cookie(name, cookie_dict[name]))
return cookiejar
libs/requests/defaults.py
+2
View File
@@ -20,6 +20,7 @@ Configurations:
:pool_connections: The number of active HTTP connection pools to use.
:encode_uri: If true, URIs will automatically be percent-encoded.
:trust_env: If true, the surrouding environment will be trusted (environ, netrc).
:param store_cookies: If false, the received cookies as part of the HTTP response would be ignored.
"""
@@ -47,5 +48,6 @@ defaults['strict_mode'] = False
defaults['keep_alive'] = True
defaults['encode_uri'] = True
defaults['trust_env'] = True
defaults['store_cookies'] = True
libs/requests/exceptions.py
+4 -1
View File
@@ -35,4 +35,7 @@ class MissingSchema(RequestException, ValueError):
"""The URL schema (e.g. http or https) is missing."""
class InvalidSchema(RequestException, ValueError):
"""See defaults.py for valid schemas."""
"""See defaults.py for valid schemas."""
class InvalidURL(RequestException, ValueError):
""" The URL provided was somehow invalid. """
libs/requests/hooks.py
+8 -3
View File
@@ -12,6 +12,9 @@ Available hooks:
A dictionary of the arguments being sent to Request().
``pre_request``:
The Request object, directly after being created.
``pre_send``:
The Request object, directly before being sent.
``post_request``:
@@ -25,8 +28,7 @@ Available hooks:
import traceback
HOOKS = ('args', 'pre_request', 'post_request', 'response')
HOOKS = ('args', 'pre_request', 'pre_send', 'post_request', 'response')
def dispatch_hook(key, hooks, hook_data):
"""Dispatches a hook dictionary on a given piece of data."""
@@ -41,7 +43,10 @@ def dispatch_hook(key, hooks, hook_data):
for hook in hooks:
try:
hook_data = hook(hook_data) or hook_data
_hook_data = hook(hook_data)
if _hook_data is not None:
hook_data = _hook_data
except Exception:
traceback.print_exc()
libs/requests/models.py
+189 -169
View File
@@ -7,6 +7,7 @@ requests.models
This module contains the primary objects that power Requests.
"""
import json
import os
from datetime import datetime
@@ -15,8 +16,8 @@ from .structures import CaseInsensitiveDict
from .status_codes import codes
from .auth import HTTPBasicAuth, HTTPProxyAuth
from .packages.urllib3.response import HTTPResponse
from .packages.urllib3.exceptions import MaxRetryError
from .cookies import cookiejar_from_dict, extract_cookies_to_jar, get_cookie_header
from .packages.urllib3.exceptions import MaxRetryError, LocationParseError
from .packages.urllib3.exceptions import SSLError as _SSLError
from .packages.urllib3.exceptions import HTTPError as _HTTPError
from .packages.urllib3 import connectionpool, poolmanager
@@ -24,22 +25,25 @@ from .packages.urllib3.filepost import encode_multipart_formdata
from .defaults import SCHEMAS
from .exceptions import (
ConnectionError, HTTPError, RequestException, Timeout, TooManyRedirects,
URLRequired, SSLError, MissingSchema, InvalidSchema)
URLRequired, SSLError, MissingSchema, InvalidSchema, InvalidURL)
from .utils import (
get_encoding_from_headers, stream_untransfer, guess_filename, requote_uri,
dict_from_string, stream_decode_response_unicode, get_netrc_auth)
stream_decode_response_unicode, get_netrc_auth, get_environ_proxies,
DEFAULT_CA_BUNDLE_PATH)
from .compat import (
urlparse, urlunparse, urljoin, urlsplit, urlencode, str, bytes,
SimpleCookie, is_py2)
cookielib, urlparse, urlunparse, urljoin, urlsplit, urlencode, str, bytes,
StringIO, is_py2)
# Import chardet if it is available.
try:
import chardet
# hush pyflakes
chardet
except ImportError:
pass
chardet = None
REDIRECT_STATI = (codes.moved, codes.found, codes.other, codes.temporary_moved)
CONTENT_CHUNK_SIZE = 10 * 1024
class Request(object):
"""The :class:`Request <Request>` object. It carries out all functionality of
@@ -61,6 +65,7 @@ class Request(object):
proxies=None,
hooks=None,
config=None,
prefetch=False,
_poolmanager=None,
verify=None,
session=None,
@@ -80,17 +85,18 @@ class Request(object):
self.headers = dict(headers or [])
#: Dictionary of files to multipart upload (``{filename: content}``).
self.files = files
self.files = None
#: HTTP Method to use.
self.method = method
#: Dictionary or byte of request body data to attach to the
#: Dictionary, bytes or file stream of request body data to attach to the
#: :class:`Request <Request>`.
self.data = None
#: Dictionary or byte of querystring data to attach to the
#: :class:`Request <Request>`.
#: :class:`Request <Request>`. The dictionary values can be lists for representing
#: multivalued query parameters.
self.params = None
#: True if :class:`Request <Request>` is part of a redirect chain (disables history
@@ -106,13 +112,11 @@ class Request(object):
# If no proxies are given, allow configuration by environment variables
# HTTP_PROXY and HTTPS_PROXY.
if not self.proxies and self.config.get('trust_env'):
if 'HTTP_PROXY' in os.environ:
self.proxies['http'] = os.environ['HTTP_PROXY']
if 'HTTPS_PROXY' in os.environ:
self.proxies['https'] = os.environ['HTTPS_PROXY']
self.proxies = get_environ_proxies()
self.data, self._enc_data = self._encode_params(data)
self.params, self._enc_params = self._encode_params(params)
self.data = data
self.params = params
self.files = files
#: :class:`Response <Response>` instance, containing
#: content and metadata of HTTP Response, once :attr:`sent <send>`.
@@ -122,7 +126,10 @@ class Request(object):
self.auth = auth
#: CookieJar to attach to :class:`Request <Request>`.
self.cookies = dict(cookies or [])
if isinstance(cookies, cookielib.CookieJar):
self.cookies = cookies
else:
self.cookies = cookiejar_from_dict(cookies)
#: True if Request has been sent.
self.sent = False
@@ -147,6 +154,9 @@ class Request(object):
#: SSL Certificate
self.cert = cert
#: Prefetch response content
self.prefetch = prefetch
if headers:
headers = CaseInsensitiveDict(self.headers)
else:
@@ -186,16 +196,16 @@ class Request(object):
# Set encoding.
response.encoding = get_encoding_from_headers(response.headers)
# Start off with our local cookies.
cookies = self.cookies or dict()
# Add new cookies from the server.
if 'set-cookie' in response.headers:
cookie_header = response.headers['set-cookie']
cookies = dict_from_string(cookie_header)
# Add new cookies from the server. Don't if configured not to
if self.config.get('store_cookies'):
extract_cookies_to_jar(self.cookies, self, resp)
# Save cookies in Response.
response.cookies = cookies
response.cookies = self.cookies
# Save cookies in Session.
for cookie in self.cookies:
self.session.cookies.set_cookie(cookie)
# No exceptions were harmed in the making of this request.
response.error = getattr(resp, 'error', None)
@@ -213,8 +223,6 @@ class Request(object):
r = build(resp)
self.cookies.update(r.cookies)
if r.status_code in REDIRECT_STATI and not self.redirect:
while (('location' in r.headers) and
@@ -232,6 +240,7 @@ class Request(object):
url = r.headers['location']
data = self.data
files = self.files
# Handle redirection without scheme (see: RFC 1808 Section 4)
if url.startswith('//'):
@@ -250,6 +259,7 @@ class Request(object):
if r.status_code is codes.see_other:
method = 'GET'
data = None
files = None
else:
method = self.method
@@ -259,10 +269,12 @@ class Request(object):
if r.status_code in (codes.moved, codes.found) and self.method == 'POST':
method = 'GET'
data = None
files = None
if (r.status_code == 303) and self.method != 'HEAD':
method = 'GET'
data = None
files = None
# Remove the cookie headers that were sent.
headers = self.headers
@@ -274,7 +286,7 @@ class Request(object):
request = Request(
url=url,
headers=headers,
files=self.files,
files=files,
method=method,
params=self.session.params,
auth=self.auth,
@@ -292,41 +304,68 @@ class Request(object):
request.send()
r = request.response
self.cookies.update(r.cookies)
r.history = history
self.response = r
self.response.request = self
self.response.cookies.update(self.cookies)
@staticmethod
def _encode_params(data):
"""Encode parameters in a piece of data.
If the data supplied is a dictionary, encodes each parameter in it, and
returns a list of tuples containing the encoded parameters, and a urlencoded
version of that.
Otherwise, assumes the data is already encoded appropriately, and
returns it twice.
Will successfully encode parameters when passed as a dict or a list of
2-tuples. Order is retained if data is a list of 2-tuples but abritrary
if parameters are supplied as a dict.
"""
if isinstance(data, bytes):
return data, data
return data
if isinstance(data, str):
return data
elif hasattr(data, 'read'):
return data
elif hasattr(data, '__iter__'):
try:
dict(data)
except ValueError:
raise ValueError('Unable to encode lists with elements that are not 2-tuples.')
if hasattr(data, '__iter__') and not isinstance(data, str):
data = dict(data)
if hasattr(data, 'items'):
params = list(data.items() if isinstance(data, dict) else data)
result = []
for k, vs in list(data.items()):
for k, vs in params:
for v in isinstance(vs, list) and vs or [vs]:
result.append((k.encode('utf-8') if isinstance(k, str) else k,
v.encode('utf-8') if isinstance(v, str) else v))
return result, urlencode(result, doseq=True)
result.append(
(k.encode('utf-8') if isinstance(k, str) else k,
v.encode('utf-8') if isinstance(v, str) else v))
return urlencode(result, doseq=True)
else:
return data, data
return data
def _encode_files(self, files):
if (not files) or isinstance(self.data, str):
return None
try:
fields = self.data.copy()
except AttributeError:
fields = dict(self.data)
for (k, v) in list(files.items()):
# support for explicit filename
if isinstance(v, (tuple, list)):
fn, fp = v
else:
fn = guess_filename(v) or k
fp = v
if isinstance(fp, (bytes, str)):
fp = StringIO(fp)
fields.update({k: (fn, fp.read())})
(body, content_type) = encode_multipart_formdata(fields)
return (body, content_type)
@property
def full_url(self):
@@ -351,7 +390,6 @@ class Request(object):
if not path:
path = '/'
if is_py2:
if isinstance(scheme, str):
scheme = scheme.encode('utf-8')
@@ -368,11 +406,12 @@ class Request(object):
url = (urlunparse([scheme, netloc, path, params, query, fragment]))
if self._enc_params:
enc_params = self._encode_params(self.params)
if enc_params:
if urlparse(url).query:
url = '%s&%s' % (url, self._enc_params)
url = '%s&%s' % (url, enc_params)
else:
url = '%s?%s' % (url, self._enc_params)
url = '%s?%s' % (url, enc_params)
if self.config.get('encode_uri', True):
url = requote_uri(url)
@@ -407,10 +446,21 @@ class Request(object):
def register_hook(self, event, hook):
"""Properly register a hook."""
return self.hooks[event].append(hook)
self.hooks[event].append(hook)
def deregister_hook(self, event, hook):
"""Deregister a previously registered hook.
Returns True if the hook existed, False if not.
"""
try:
self.hooks[event].remove(hook)
return True
except ValueError:
return False
def send(self, anyway=False, prefetch=False):
"""Sends the request. Returns True of successful, False if not.
"""Sends the request. Returns True if successful, False if not.
If there was an HTTPError during transmission,
self.response.status_code will contain the HTTPError code.
@@ -423,6 +473,10 @@ class Request(object):
# Build the URL
url = self.full_url
# Pre-request hook.
r = dispatch_hook('pre_request', self.hooks, self)
self.__dict__.update(r.__dict__)
# Logging
if self.config.get('verbose'):
self.config.get('verbose').write('%s %s %s\n' % (
@@ -433,41 +487,6 @@ class Request(object):
body = None
content_type = None
# Multi-part file uploads.
if self.files:
if not isinstance(self.data, str):
try:
fields = self.data.copy()
except AttributeError:
fields = dict(self.data)
for (k, v) in list(self.files.items()):
# support for explicit filename
if isinstance(v, (tuple, list)):
fn, fp = v
else:
fn = guess_filename(v) or k
fp = v
fields.update({k: (fn, fp.read())})
(body, content_type) = encode_multipart_formdata(fields)
else:
pass
# TODO: Conflict?
else:
if self.data:
body = self._enc_data
if isinstance(self.data, str):
content_type = None
else:
content_type = 'application/x-www-form-urlencoded'
# Add content-type if it wasn't explicitly provided.
if (content_type) and (not 'content-type' in self.headers):
self.headers['Content-Type'] = content_type
# Use .netrc auth if none was provided.
if not self.auth and self.config.get('trust_env'):
self.auth = get_netrc_auth(url)
@@ -483,10 +502,27 @@ class Request(object):
# Update self to reflect the auth changes.
self.__dict__.update(r.__dict__)
# Multi-part file uploads.
if self.files:
(body, content_type) = self._encode_files(self.files)
else:
if self.data:
body = self._encode_params(self.data)
if isinstance(self.data, str) or hasattr(self.data, 'read'):
content_type = None
else:
content_type = 'application/x-www-form-urlencoded'
# Add content-type if it wasn't explicitly provided.
if (content_type) and (not 'content-type' in self.headers):
self.headers['Content-Type'] = content_type
_p = urlparse(url)
no_proxy = filter(lambda x:x.strip(), self.proxies.get('no', '').split(','))
proxy = self.proxies.get(_p.scheme)
if proxy:
if proxy and not any(map(_p.netloc.endswith, no_proxy)):
conn = poolmanager.proxy_from_url(proxy)
_proxy = urlparse(proxy)
if '@' in _proxy.netloc:
@@ -496,10 +532,14 @@ class Request(object):
self.__dict__.update(r.__dict__)
else:
# Check to see if keep_alive is allowed.
if self.config.get('keep_alive'):
conn = self._poolmanager.connection_from_url(url)
else:
conn = connectionpool.connection_from_url(url)
try:
if self.config.get('keep_alive'):
conn = self._poolmanager.connection_from_url(url)
else:
conn = connectionpool.connection_from_url(url)
self.headers['Connection'] = 'close'
except LocationParseError as e:
raise InvalidURL(e)
if url.startswith('https') and self.verify:
@@ -513,13 +553,15 @@ class Request(object):
if not cert_loc and self.config.get('trust_env'):
cert_loc = os.environ.get('REQUESTS_CA_BUNDLE')
# Curl compatiblity.
# Curl compatibility.
if not cert_loc and self.config.get('trust_env'):
cert_loc = os.environ.get('CURL_CA_BUNDLE')
# Use the awesome certifi list.
if not cert_loc:
cert_loc = __import__('certifi').where()
cert_loc = DEFAULT_CA_BUNDLE_PATH
if not cert_loc:
raise Exception("Could not find a suitable SSL CA certificate bundle.")
conn.cert_reqs = 'CERT_REQUIRED'
conn.ca_certs = cert_loc
@@ -536,64 +578,43 @@ class Request(object):
if not self.sent or anyway:
if self.cookies:
# Skip if 'cookie' header is explicitly set.
if 'cookie' not in self.headers:
# Simple cookie with our dict.
c = SimpleCookie()
for (k, v) in list(self.cookies.items()):
c[k] = v
# Turn it into a header.
cookie_header = c.output(header='', sep='; ').strip()
# Attach Cookie header to request.
# Skip if 'cookie' header is explicitly set.
if 'cookie' not in self.headers:
cookie_header = get_cookie_header(self.cookies, self)
if cookie_header is not None:
self.headers['Cookie'] = cookie_header
# Pre-request hook.
r = dispatch_hook('pre_request', self.hooks, self)
# Pre-send hook.
r = dispatch_hook('pre_send', self.hooks, self)
self.__dict__.update(r.__dict__)
# catch urllib3 exceptions and throw Requests exceptions
try:
# The inner try .. except re-raises certain exceptions as
# internal exception types; the outer suppresses exceptions
# when safe mode is set.
try:
# Send the request.
r = conn.urlopen(
method=self.method,
url=self.path_url,
body=body,
headers=self.headers,
redirect=False,
assert_same_host=False,
preload_content=False,
decode_content=False,
retries=self.config.get('max_retries', 0),
timeout=self.timeout,
)
self.sent = True
# Send the request.
r = conn.urlopen(
method=self.method,
url=self.path_url,
body=body,
headers=self.headers,
redirect=False,
assert_same_host=False,
preload_content=False,
decode_content=False,
retries=self.config.get('max_retries', 0),
timeout=self.timeout,
)
self.sent = True
except MaxRetryError as e:
raise ConnectionError(e)
except MaxRetryError as e:
raise ConnectionError(e)
except (_SSLError, _HTTPError) as e:
if self.verify and isinstance(e, _SSLError):
raise SSLError(e)
except (_SSLError, _HTTPError) as e:
if self.verify and isinstance(e, _SSLError):
raise SSLError(e)
raise Timeout('Request timed out.')
except RequestException as e:
if self.config.get('safe_mode', False):
# In safe mode, catch the exception and attach it to
# a blank urllib3.HTTPResponse object.
r = HTTPResponse()
r.error = e
else:
raise
raise Timeout('Request timed out.')
# build_response can throw TooManyRedirects
self._build_response(r)
# Response manipulation hook.
@@ -604,7 +625,7 @@ class Request(object):
self.__dict__.update(r.__dict__)
# If prefetch is True, mark content as consumed.
if prefetch:
if prefetch or self.prefetch:
# Save the response.
self.response.content
@@ -623,7 +644,7 @@ class Response(object):
def __init__(self):
self._content = None
self._content = False
self._content_consumed = False
#: Integer Code of responded HTTP Status.
@@ -643,7 +664,7 @@ class Response(object):
#: Resulting :class:`HTTPError` of request, if one occurred.
self.error = None
#: Encoding to decode with when accessing r.content.
#: Encoding to decode with when accessing r.text.
self.encoding = None
#: A list of :class:`Response <Response>` objects from
@@ -654,8 +675,8 @@ class Response(object):
#: The :class:`Request <Request>` that created the Response.
self.request = None
#: A dictionary of Cookies the server sent back.
self.cookies = {}
#: A CookieJar of Cookies the server sent back.
self.cookies = None
#: Dictionary of configurations for this request.
self.config = {}
@@ -679,7 +700,7 @@ class Response(object):
return False
return True
def iter_content(self, chunk_size=10 * 1024, decode_unicode=False):
def iter_content(self, chunk_size=1, decode_unicode=False):
"""Iterates over the response data. This avoids reading the content
at once into memory for large responses. The chunk size is the number
of bytes it should read into memory. This is not necessarily the
@@ -721,7 +742,7 @@ class Response(object):
chunk = pending + chunk
lines = chunk.splitlines()
if lines[-1][-1] == chunk[-1]:
if lines and lines[-1] and chunk and lines[-1][-1] == chunk[-1]:
pending = lines.pop()
else:
pending = None
@@ -736,7 +757,7 @@ class Response(object):
def content(self):
"""Content of the response, in bytes."""
if self._content is None:
if self._content is False:
# Read the contents.
try:
if self._content_consumed:
@@ -746,7 +767,7 @@ class Response(object):
if self.status_code is 0:
self._content = None
else:
self._content = bytes().join(self.iter_content()) or bytes()
self._content = bytes().join(self.iter_content(CONTENT_CHUNK_SIZE)) or bytes()
except AttributeError:
self._content = None
@@ -754,16 +775,6 @@ class Response(object):
self._content_consumed = True
return self._content
def _detected_encoding(self):
try:
detected = chardet.detect(self.content) or {}
return detected.get('encoding')
# Trust that chardet isn't available or something went terribly wrong.
except Exception:
pass
@property
def text(self):
"""Content of the response, in unicode.
@@ -776,9 +787,13 @@ class Response(object):
content = None
encoding = self.encoding
# Fallback to auto-detected encoding if chardet is available.
if not self.content:
return str('')
# Fallback to auto-detected encoding.
if self.encoding is None:
encoding = self._detected_encoding()
if chardet is not None:
encoding = chardet.detect(self.content)['encoding']
# Decode unicode from given encoding.
try:
@@ -789,11 +804,17 @@ class Response(object):
#
# So we try blindly encoding.
content = str(self.content, errors='replace')
except (UnicodeError, TypeError):
pass
return content
@property
def json(self):
"""Returns the json-encoded content of a request, if any."""
try:
return json.loads(self.text or self.content)
except ValueError:
return None
def raise_for_status(self, allow_redirects=True):
"""Raises stored :class:`HTTPError` or :class:`URLError`, if one occurred."""
@@ -810,7 +831,6 @@ class Response(object):
http_error.response = self
raise http_error
elif (self.status_code >= 500) and (self.status_code < 600):
http_error = HTTPError('%s Server Error' % self.status_code)
http_error.response = self
libs/requests/packages/oreos/__init__.py
-3
View File
@@ -1,3 +0,0 @@
# -*- coding: utf-8 -*-
from .core import dict_from_string
libs/requests/packages/oreos/core.py
-24
View File
@@ -1,24 +0,0 @@
# -*- coding: utf-8 -*-
"""
oreos.core
~~~~~~~~~~
The creamy white center.
"""
from .monkeys import SimpleCookie
def dict_from_string(s):
"""Returns a MultiDict with Cookies."""
cookies = dict()
c = SimpleCookie()
c.load(s)
for k,v in c.items():
cookies.update({k: v.value})
return cookies
libs/requests/packages/oreos/monkeys.py
-773
View File
@@ -1,773 +0,0 @@
# -*- coding: utf-8 -*-
"""
oreos.monkeys
~~~~~~~~~~~~~
Monkeypatches.
"""
#!/usr/bin/env python
#
####
# Copyright 2000 by Timothy O'Malley <timo@alum.mit.edu>
#
# All Rights Reserved
#
# Permission to use, copy, modify, and distribute this software
# and its documentation for any purpose and without fee is hereby
# granted, provided that the above copyright notice appear in all
# copies and that both that copyright notice and this permission
# notice appear in supporting documentation, and that the name of
# Timothy O'Malley not be used in advertising or publicity
# pertaining to distribution of the software without specific, written
# prior permission.
#
# Timothy O'Malley DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS
# SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
# AND FITNESS, IN NO EVENT SHALL Timothy O'Malley BE LIABLE FOR
# ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
# WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS
# ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
# PERFORMANCE OF THIS SOFTWARE.
#
####
#
# Id: Cookie.py,v 2.29 2000/08/23 05:28:49 timo Exp
# by Timothy O'Malley <timo@alum.mit.edu>
#
# Cookie.py is a Python module for the handling of HTTP
# cookies as a Python dictionary. See RFC 2109 for more
# information on cookies.
#
# The original idea to treat Cookies as a dictionary came from
# Dave Mitchell (davem@magnet.com) in 1995, when he released the
# first version of nscookie.py.
#
####
r"""
Here's a sample session to show how to use this module.
At the moment, this is the only documentation.
The Basics
----------
Importing is easy..
>>> import Cookie
Most of the time you start by creating a cookie. Cookies come in
three flavors, each with slightly different encoding semantics, but
more on that later.
>>> C = Cookie.SimpleCookie()
>>> C = Cookie.SerialCookie()
>>> C = Cookie.SmartCookie()
[Note: Long-time users of Cookie.py will remember using
Cookie.Cookie() to create an Cookie object. Although deprecated, it
is still supported by the code. See the Backward Compatibility notes
for more information.]
Once you've created your Cookie, you can add values just as if it were
a dictionary.
>>> C = Cookie.SmartCookie()
>>> C["fig"] = "newton"
>>> C["sugar"] = "wafer"
>>> C.output()
'Set-Cookie: fig=newton\r\nSet-Cookie: sugar=wafer'
Notice that the printable representation of a Cookie is the
appropriate format for a Set-Cookie: header. This is the
default behavior. You can change the header and printed
attributes by using the .output() function
>>> C = Cookie.SmartCookie()
>>> C["rocky"] = "road"
>>> C["rocky"]["path"] = "/cookie"
>>> print C.output(header="Cookie:")
Cookie: rocky=road; Path=/cookie
>>> print C.output(attrs=[], header="Cookie:")
Cookie: rocky=road
The load() method of a Cookie extracts cookies from a string. In a
CGI script, you would use this method to extract the cookies from the
HTTP_COOKIE environment variable.
>>> C = Cookie.SmartCookie()
>>> C.load("chips=ahoy; vienna=finger")
>>> C.output()
'Set-Cookie: chips=ahoy\r\nSet-Cookie: vienna=finger'
The load() method is darn-tootin smart about identifying cookies
within a string. Escaped quotation marks, nested semicolons, and other
such trickeries do not confuse it.
>>> C = Cookie.SmartCookie()
>>> C.load('keebler="E=everybody; L=\\"Loves\\"; fudge=\\012;";')
>>> print C
Set-Cookie: keebler="E=everybody; L=\"Loves\"; fudge=\012;"
Each element of the Cookie also supports all of the RFC 2109
Cookie attributes. Here's an example which sets the Path
attribute.
>>> C = Cookie.SmartCookie()
>>> C["oreo"] = "doublestuff"
>>> C["oreo"]["path"] = "/"
>>> print C
Set-Cookie: oreo=doublestuff; Path=/
Each dictionary element has a 'value' attribute, which gives you
back the value associated with the key.
>>> C = Cookie.SmartCookie()
>>> C["twix"] = "none for you"
>>> C["twix"].value
'none for you'
A Bit More Advanced
-------------------
As mentioned before, there are three different flavors of Cookie
objects, each with different encoding/decoding semantics. This
section briefly discusses the differences.
SimpleCookie
The SimpleCookie expects that all values should be standard strings.
Just to be sure, SimpleCookie invokes the str() builtin to convert
the value to a string, when the values are set dictionary-style.
>>> C = Cookie.SimpleCookie()
>>> C["number"] = 7
>>> C["string"] = "seven"
>>> C["number"].value
'7'
>>> C["string"].value
'seven'
>>> C.output()
'Set-Cookie: number=7\r\nSet-Cookie: string=seven'
SerialCookie
The SerialCookie expects that all values should be serialized using
cPickle (or pickle, if cPickle isn't available). As a result of
serializing, SerialCookie can save almost any Python object to a
value, and recover the exact same object when the cookie has been
returned. (SerialCookie can yield some strange-looking cookie
values, however.)
>>> C = Cookie.SerialCookie()
>>> C["number"] = 7
>>> C["string"] = "seven"
>>> C["number"].value
7
>>> C["string"].value
'seven'
>>> C.output()
'Set-Cookie: number="I7\\012."\r\nSet-Cookie: string="S\'seven\'\\012p1\\012."'
Be warned, however, if SerialCookie cannot de-serialize a value (because
it isn't a valid pickle'd object), IT WILL RAISE AN EXCEPTION.
SmartCookie
The SmartCookie combines aspects of each of the other two flavors.
When setting a value in a dictionary-fashion, the SmartCookie will
serialize (ala cPickle) the value *if and only if* it isn't a
Python string. String objects are *not* serialized. Similarly,
when the load() method parses out values, it attempts to de-serialize
the value. If it fails, then it fallsback to treating the value
as a string.
>>> C = Cookie.SmartCookie()
>>> C["number"] = 7
>>> C["string"] = "seven"
>>> C["number"].value
7
>>> C["string"].value
'seven'
>>> C.output()
'Set-Cookie: number="I7\\012."\r\nSet-Cookie: string=seven'
Backwards Compatibility
-----------------------
In order to keep compatibilty with earlier versions of Cookie.py,
it is still possible to use Cookie.Cookie() to create a Cookie. In
fact, this simply returns a SmartCookie.
>>> C = Cookie.Cookie()
>>> print C.__class__.__name__
SmartCookie
Finis.
""" #"
# ^
# |----helps out font-lock
#
# Import our required modules
#
import string
try:
from cPickle import dumps, loads
except ImportError:
from pickle import dumps, loads
import re, warnings
__all__ = ["CookieError","BaseCookie","SimpleCookie","SerialCookie",
"SmartCookie","Cookie"]
_nulljoin = ''.join
_semispacejoin = '; '.join
_spacejoin = ' '.join
#
# Define an exception visible to External modules
#
class CookieError(Exception):
pass
# These quoting routines conform to the RFC2109 specification, which in
# turn references the character definitions from RFC2068. They provide
# a two-way quoting algorithm. Any non-text character is translated
# into a 4 character sequence: a forward-slash followed by the
# three-digit octal equivalent of the character. Any '\' or '"' is
# quoted with a preceeding '\' slash.
#
# These are taken from RFC2068 and RFC2109.
# _RFC2965Forbidden is the list of forbidden chars we accept anyway
# _LegalChars is the list of chars which don't require "'s
# _Translator hash-table for fast quoting
#
_RFC2965Forbidden = "[]:{}="
_LegalChars = ( string.ascii_letters + string.digits +
"!#$%&'*+-.^_`|~_@" + _RFC2965Forbidden )
_Translator = {
'\000' : '\\000', '\001' : '\\001', '\002' : '\\002',
'\003' : '\\003', '\004' : '\\004', '\005' : '\\005',
'\006' : '\\006', '\007' : '\\007', '\010' : '\\010',
'\011' : '\\011', '\012' : '\\012', '\013' : '\\013',
'\014' : '\\014', '\015' : '\\015', '\016' : '\\016',
'\017' : '\\017', '\020' : '\\020', '\021' : '\\021',
'\022' : '\\022', '\023' : '\\023', '\024' : '\\024',
'\025' : '\\025', '\026' : '\\026', '\027' : '\\027',
'\030' : '\\030', '\031' : '\\031', '\032' : '\\032',
'\033' : '\\033', '\034' : '\\034', '\035' : '\\035',
'\036' : '\\036', '\037' : '\\037',
# Because of the way browsers really handle cookies (as opposed
# to what the RFC says) we also encode , and ;
',' : '\\054', ';' : '\\073',
'"' : '\\"', '\\' : '\\\\',
'\177' : '\\177', '\200' : '\\200', '\201' : '\\201',
'\202' : '\\202', '\203' : '\\203', '\204' : '\\204',
'\205' : '\\205', '\206' : '\\206', '\207' : '\\207',
'\210' : '\\210', '\211' : '\\211', '\212' : '\\212',
'\213' : '\\213', '\214' : '\\214', '\215' : '\\215',
'\216' : '\\216', '\217' : '\\217', '\220' : '\\220',
'\221' : '\\221', '\222' : '\\222', '\223' : '\\223',
'\224' : '\\224', '\225' : '\\225', '\226' : '\\226',
'\227' : '\\227', '\230' : '\\230', '\231' : '\\231',
'\232' : '\\232', '\233' : '\\233', '\234' : '\\234',
'\235' : '\\235', '\236' : '\\236', '\237' : '\\237',
'\240' : '\\240', '\241' : '\\241', '\242' : '\\242',
'\243' : '\\243', '\244' : '\\244', '\245' : '\\245',
'\246' : '\\246', '\247' : '\\247', '\250' : '\\250',
'\251' : '\\251', '\252' : '\\252', '\253' : '\\253',
'\254' : '\\254', '\255' : '\\255', '\256' : '\\256',
'\257' : '\\257', '\260' : '\\260', '\261' : '\\261',
'\262' : '\\262', '\263' : '\\263', '\264' : '\\264',
'\265' : '\\265', '\266' : '\\266', '\267' : '\\267',
'\270' : '\\270', '\271' : '\\271', '\272' : '\\272',
'\273' : '\\273', '\274' : '\\274', '\275' : '\\275',
'\276' : '\\276', '\277' : '\\277', '\300' : '\\300',
'\301' : '\\301', '\302' : '\\302', '\303' : '\\303',
'\304' : '\\304', '\305' : '\\305', '\306' : '\\306',
'\307' : '\\307', '\310' : '\\310', '\311' : '\\311',
'\312' : '\\312', '\313' : '\\313', '\314' : '\\314',
'\315' : '\\315', '\316' : '\\316', '\317' : '\\317',
'\320' : '\\320', '\321' : '\\321', '\322' : '\\322',
'\323' : '\\323', '\324' : '\\324', '\325' : '\\325',
'\326' : '\\326', '\327' : '\\327', '\330' : '\\330',
'\331' : '\\331', '\332' : '\\332', '\333' : '\\333',
'\334' : '\\334', '\335' : '\\335', '\336' : '\\336',
'\337' : '\\337', '\340' : '\\340', '\341' : '\\341',
'\342' : '\\342', '\343' : '\\343', '\344' : '\\344',
'\345' : '\\345', '\346' : '\\346', '\347' : '\\347',
'\350' : '\\350', '\351' : '\\351', '\352' : '\\352',
'\353' : '\\353', '\354' : '\\354', '\355' : '\\355',
'\356' : '\\356', '\357' : '\\357', '\360' : '\\360',
'\361' : '\\361', '\362' : '\\362', '\363' : '\\363',
'\364' : '\\364', '\365' : '\\365', '\366' : '\\366',
'\367' : '\\367', '\370' : '\\370', '\371' : '\\371',
'\372' : '\\372', '\373' : '\\373', '\374' : '\\374',
'\375' : '\\375', '\376' : '\\376', '\377' : '\\377'
}
_idmap = ''.join(chr(x) for x in range(256))
def _quote(str, LegalChars=_LegalChars,
idmap=_idmap, translate=string.translate):
#
# If the string does not need to be double-quoted,
# then just return the string. Otherwise, surround
# the string in doublequotes and precede quote (with a \)
# special characters.
#
if "" == translate(str, idmap, LegalChars):
return str
else:
return '"' + _nulljoin( map(_Translator.get, str, str) ) + '"'
# end _quote
_OctalPatt = re.compile(r"\\[0-3][0-7][0-7]")
_QuotePatt = re.compile(r"[\\].")
def _unquote(str):
# If there aren't any doublequotes,
# then there can't be any special characters. See RFC 2109.
if len(str) < 2:
return str
if str[0] != '"' or str[-1] != '"':
return str
# We have to assume that we must decode this string.
# Down to work.
# Remove the "s
str = str[1:-1]
# Check for special sequences. Examples:
# \012 --> \n
# \" --> "
#
i = 0
n = len(str)
res = []
while 0 <= i < n:
Omatch = _OctalPatt.search(str, i)
Qmatch = _QuotePatt.search(str, i)
if not Omatch and not Qmatch: # Neither matched
res.append(str[i:])
break
# else:
j = k = -1
if Omatch: j = Omatch.start(0)
if Qmatch: k = Qmatch.start(0)
if Qmatch and ( not Omatch or k < j ): # QuotePatt matched
res.append(str[i:k])
res.append(str[k+1])
i = k+2
else: # OctalPatt matched
res.append(str[i:j])
res.append( chr( int(str[j+1:j+4], 8) ) )
i = j+4
return _nulljoin(res)
# end _unquote
# The _getdate() routine is used to set the expiration time in
# the cookie's HTTP header. By default, _getdate() returns the
# current time in the appropriate "expires" format for a
# Set-Cookie header. The one optional argument is an offset from
# now, in seconds. For example, an offset of -3600 means "one hour ago".
# The offset may be a floating point number.
#
_weekdayname = ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun']
_monthname = [None,
'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun',
'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']
def _getdate(future=0, weekdayname=_weekdayname, monthname=_monthname):
from time import gmtime, time
now = time()
year, month, day, hh, mm, ss, wd, y, z = gmtime(now + future)
return "%s, %02d-%3s-%4d %02d:%02d:%02d GMT" % \
(weekdayname[wd], day, monthname[month], year, hh, mm, ss)
#
# A class to hold ONE key,value pair.
# In a cookie, each such pair may have several attributes.
# so this class is used to keep the attributes associated
# with the appropriate key,value pair.
# This class also includes a coded_value attribute, which
# is used to hold the network representation of the
# value. This is most useful when Python objects are
# pickled for network transit.
#
class Morsel(dict):
# RFC 2109 lists these attributes as reserved:
# path comment domain
# max-age secure version
#
# For historical reasons, these attributes are also reserved:
# expires
#
# This is an extension from Microsoft:
# httponly
#
# This dictionary provides a mapping from the lowercase
# variant on the left to the appropriate traditional
# formatting on the right.
_reserved = { "expires" : "expires",
"path" : "Path",
"comment" : "Comment",
"domain" : "Domain",
"max-age" : "Max-Age",
"secure" : "secure",
"httponly" : "httponly",
"version" : "Version",
}
def __init__(self):
# Set defaults
self.key = self.value = self.coded_value = None
# Set default attributes
for K in self._reserved:
dict.__setitem__(self, K, "")
# end __init__
def __setitem__(self, K, V):
K = K.lower()
if not K in self._reserved:
raise CookieError("Invalid Attribute %s" % K)
dict.__setitem__(self, K, V)
# end __setitem__
def isReservedKey(self, K):
return K.lower() in self._reserved
# end isReservedKey
def set(self, key, val, coded_val,
LegalChars=_LegalChars,
idmap=_idmap, translate=string.translate):
# First we verify that the key isn't a reserved word
# Second we make sure it only contains legal characters
if key.lower() in self._reserved:
raise CookieError("Attempt to set a reserved key: %s" % key)
if "" != translate(key, idmap, LegalChars):
raise CookieError("Illegal key value: %s" % key)
# It's a good key, so save it.
self.key = key
self.value = val
self.coded_value = coded_val
# end set
def output(self, attrs=None, header = "Set-Cookie:"):
return "%s %s" % ( header, self.OutputString(attrs) )
__str__ = output
def __repr__(self):
return '<%s: %s=%s>' % (self.__class__.__name__,
self.key, repr(self.value) )
def js_output(self, attrs=None):
# Print javascript
return """
<script type="text/javascript">
<!-- begin hiding
document.cookie = \"%s\";
// end hiding -->
</script>
""" % ( self.OutputString(attrs).replace('"',r'\"'), )
# end js_output()
def OutputString(self, attrs=None):
# Build up our result
#
result = []
RA = result.append
# First, the key=value pair
RA("%s=%s" % (self.key, self.coded_value))
# Now add any defined attributes
if attrs is None:
attrs = self._reserved
items = self.items()
items.sort()
for K,V in items:
if V == "": continue
if K not in attrs: continue
if K == "expires" and type(V) == type(1):
RA("%s=%s" % (self._reserved[K], _getdate(V)))
elif K == "max-age" and type(V) == type(1):
RA("%s=%d" % (self._reserved[K], V))
elif K == "secure":
RA(str(self._reserved[K]))
elif K == "httponly":
RA(str(self._reserved[K]))
else:
RA("%s=%s" % (self._reserved[K], V))
# Return the result
return _semispacejoin(result)
# end OutputString
# end Morsel class
#
# Pattern for finding cookie
#
# This used to be strict parsing based on the RFC2109 and RFC2068
# specifications. I have since discovered that MSIE 3.0x doesn't
# follow the character rules outlined in those specs. As a
# result, the parsing rules here are less strict.
#
_LegalCharsPatt = r"[\w\d!#%&'~_`><@,:/\$\*\+\-\.\^\|\)\(\?\}\{\=\[\]\_]"
_CookiePattern = re.compile(
r"(?x)" # This is a Verbose pattern
r"(?P<key>" # Start of group 'key'
""+ _LegalCharsPatt +"+?" # Any word of at least one letter, nongreedy
r")" # End of group 'key'
r"\s*=\s*" # Equal Sign
r"(?P<val>" # Start of group 'val'
r'"(?:[^\\"]|\\.)*"' # Any doublequoted string
r"|" # or
r"\w{3},\s[\w\d-]{9,11}\s[\d:]{8}\sGMT" # Special case for "expires" attr
r"|" # or
""+ _LegalCharsPatt +"*" # Any word or empty string
r")" # End of group 'val'
r"\s*;?" # Probably ending in a semi-colon
)
# At long last, here is the cookie class.
# Using this class is almost just like using a dictionary.
# See this module's docstring for example usage.
#
class BaseCookie(dict):
# A container class for a set of Morsels
#
def value_decode(self, val):
"""real_value, coded_value = value_decode(STRING)
Called prior to setting a cookie's value from the network
representation. The VALUE is the value read from HTTP
header.
Override this function to modify the behavior of cookies.
"""
return val, val
# end value_encode
def value_encode(self, val):
"""real_value, coded_value = value_encode(VALUE)
Called prior to setting a cookie's value from the dictionary
representation. The VALUE is the value being assigned.
Override this function to modify the behavior of cookies.
"""
strval = str(val)
return strval, strval
# end value_encode
def __init__(self, input=None):
if input: self.load(input)
# end __init__
def __set(self, key, real_value, coded_value):
"""Private method for setting a cookie's value"""
M = self.get(key, Morsel())
M.set(key, real_value, coded_value)
dict.__setitem__(self, key, M)
# end __set
def __setitem__(self, key, value):
"""Dictionary style assignment."""
rval, cval = self.value_encode(value)
self.__set(key, rval, cval)
# end __setitem__
def output(self, attrs=None, header="Set-Cookie:", sep="\015\012"):
"""Return a string suitable for HTTP."""
result = []
items = self.items()
items.sort()
for K,V in items:
result.append( V.output(attrs, header) )
return sep.join(result)
# end output
__str__ = output
def __repr__(self):
L = []
items = self.items()
items.sort()
for K,V in items:
L.append( '%s=%s' % (K,repr(V.value) ) )
return '<%s: %s>' % (self.__class__.__name__, _spacejoin(L))
def js_output(self, attrs=None):
"""Return a string suitable for JavaScript."""
result = []
items = self.items()
items.sort()
for K,V in items:
result.append( V.js_output(attrs) )
return _nulljoin(result)
# end js_output
def load(self, rawdata):
"""Load cookies from a string (presumably HTTP_COOKIE) or
from a dictionary. Loading cookies from a dictionary 'd'
is equivalent to calling:
map(Cookie.__setitem__, d.keys(), d.values())
"""
if type(rawdata) == type(""):
self.__ParseString(rawdata)
else:
# self.update() wouldn't call our custom __setitem__
for k, v in rawdata.items():
self[k] = v
return
# end load()
def __ParseString(self, str, patt=_CookiePattern):
i = 0 # Our starting point
n = len(str) # Length of string
M = None # current morsel
while 0 <= i < n:
# Start looking for a cookie
match = patt.search(str, i)
if not match: break # No more cookies
K,V = match.group("key"), match.group("val")
i = match.end(0)
# Parse the key, value in case it's metainfo
if K[0] == "$":
# We ignore attributes which pertain to the cookie
# mechanism as a whole. See RFC 2109.
# (Does anyone care?)
if M:
M[ K[1:] ] = V
elif K.lower() in Morsel._reserved:
if M:
M[ K ] = _unquote(V)
else:
rval, cval = self.value_decode(V)
self.__set(K, rval, cval)
M = self[K]
# end __ParseString
# end BaseCookie class
class SimpleCookie(BaseCookie):
"""SimpleCookie
SimpleCookie supports strings as cookie values. When setting
the value using the dictionary assignment notation, SimpleCookie
calls the builtin str() to convert the value to a string. Values
received from HTTP are kept as strings.
"""
def value_decode(self, val):
return _unquote( val ), val
def value_encode(self, val):
strval = str(val)
return strval, _quote( strval )
# end SimpleCookie
class SerialCookie(BaseCookie):
"""SerialCookie
SerialCookie supports arbitrary objects as cookie values. All
values are serialized (using cPickle) before being sent to the
client. All incoming values are assumed to be valid Pickle
representations. IF AN INCOMING VALUE IS NOT IN A VALID PICKLE
FORMAT, THEN AN EXCEPTION WILL BE RAISED.
Note: Large cookie values add overhead because they must be
retransmitted on every HTTP transaction.
Note: HTTP has a 2k limit on the size of a cookie. This class
does not check for this limit, so be careful!!!
"""
def __init__(self, input=None):
warnings.warn("SerialCookie class is insecure; do not use it",
DeprecationWarning)
BaseCookie.__init__(self, input)
# end __init__
def value_decode(self, val):
# This could raise an exception!
return loads( _unquote(val) ), val
def value_encode(self, val):
return val, _quote( dumps(val) )
# end SerialCookie
class SmartCookie(BaseCookie):
"""SmartCookie
SmartCookie supports arbitrary objects as cookie values. If the
object is a string, then it is quoted. If the object is not a
string, however, then SmartCookie will use cPickle to serialize
the object into a string representation.
Note: Large cookie values add overhead because they must be
retransmitted on every HTTP transaction.
Note: HTTP has a 2k limit on the size of a cookie. This class
does not check for this limit, so be careful!!!
"""
def __init__(self, input=None):
warnings.warn("Cookie/SmartCookie class is insecure; do not use it",
DeprecationWarning)
BaseCookie.__init__(self, input)
# end __init__
def value_decode(self, val):
strval = _unquote(val)
try:
return loads(strval), val
except:
return strval, val
def value_encode(self, val):
if type(val) == type(""):
return val, _quote(val)
else:
return val, _quote( dumps(val) )
# end SmartCookie
###########################################################
# Backwards Compatibility: Don't break any existing code!
# We provide Cookie() as an alias for SmartCookie()
Cookie = SmartCookie
#
###########################################################
def _test():
import doctest, Cookie
return doctest.testmod(Cookie)
if __name__ == "__main__":
_test()
#Local Variables:
#tab-width: 4
#end:
libs/requests/packages/oreos/structures.py
-399
View File
@@ -1,399 +0,0 @@
# -*- coding: utf-8 -*-
"""
oreos.structures
~~~~~~~~~~~~~~~~
The plastic blue packaging.
This is mostly directly stolen from mitsuhiko/werkzeug.
"""
__all__ = ('MultiDict',)
class _Missing(object):
def __repr__(self):
return 'no value'
def __reduce__(self):
return '_missing'
_missing = _Missing()
def iter_multi_items(mapping):
"""Iterates over the items of a mapping yielding keys and values
without dropping any from more complex structures.
"""
if isinstance(mapping, MultiDict):
for item in mapping.iteritems(multi=True):
yield item
elif isinstance(mapping, dict):
for key, value in mapping.iteritems():
if isinstance(value, (tuple, list)):
for value in value:
yield key, value
else:
yield key, value
else:
for item in mapping:
yield item
class TypeConversionDict(dict):
"""Works like a regular dict but the :meth:`get` method can perform
type conversions. :class:`MultiDict` and :class:`CombinedMultiDict`
are subclasses of this class and provide the same feature.
.. versionadded:: 0.5
"""
def get(self, key, default=None, type=None):
"""Return the default value if the requested data doesn't exist.
If `type` is provided and is a callable it should convert the value,
return it or raise a :exc:`ValueError` if that is not possible. In
this case the function will return the default as if the value was not
found:
>>> d = TypeConversionDict(foo='42', bar='blub')
>>> d.get('foo', type=int)
42
>>> d.get('bar', -1, type=int)
-1
:param key: The key to be looked up.
:param default: The default value to be returned if the key can't
be looked up. If not further specified `None` is
returned.
:param type: A callable that is used to cast the value in the
:class:`MultiDict`. If a :exc:`ValueError` is raised
by this callable the default value is returned.
"""
try:
rv = self[key]
if type is not None:
rv = type(rv)
except (KeyError, ValueError):
rv = default
return rv
class MultiDict(TypeConversionDict):
"""A :class:`MultiDict` is a dictionary subclass customized to deal with
multiple values for the same key which is for example used by the parsing
functions in the wrappers. This is necessary because some HTML form
elements pass multiple values for the same key.
:class:`MultiDict` implements all standard dictionary methods.
Internally, it saves all values for a key as a list, but the standard dict
access methods will only return the first value for a key. If you want to
gain access to the other values, too, you have to use the `list` methods as
explained below.
Basic Usage:
>>> d = MultiDict([('a', 'b'), ('a', 'c')])
>>> d
MultiDict([('a', 'b'), ('a', 'c')])
>>> d['a']
'b'
>>> d.getlist('a')
['b', 'c']
>>> 'a' in d
True
It behaves like a normal dict thus all dict functions will only return the
first value when multiple values for one key are found.
From Werkzeug 0.3 onwards, the `KeyError` raised by this class is also a
subclass of the :exc:`~exceptions.BadRequest` HTTP exception and will
render a page for a ``400 BAD REQUEST`` if caught in a catch-all for HTTP
exceptions.
A :class:`MultiDict` can be constructed from an iterable of
``(key, value)`` tuples, a dict, a :class:`MultiDict` or from Werkzeug 0.2
onwards some keyword parameters.
:param mapping: the initial value for the :class:`MultiDict`. Either a
regular dict, an iterable of ``(key, value)`` tuples
or `None`.
"""
def __init__(self, mapping=None):
if isinstance(mapping, MultiDict):
dict.__init__(self, ((k, l[:]) for k, l in mapping.iterlists()))
elif isinstance(mapping, dict):
tmp = {}
for key, value in mapping.iteritems():
if isinstance(value, (tuple, list)):
value = list(value)
else:
value = [value]
tmp[key] = value
dict.__init__(self, tmp)
else:
tmp = {}
for key, value in mapping or ():
tmp.setdefault(key, []).append(value)
dict.__init__(self, tmp)
def __getstate__(self):
return dict(self.lists())
def __setstate__(self, value):
dict.clear(self)
dict.update(self, value)
def __iter__(self):
return self.iterkeys()
def __getitem__(self, key):
"""Return the first data value for this key;
raises KeyError if not found.
:param key: The key to be looked up.
:raise KeyError: if the key does not exist.
"""
if key in self:
return dict.__getitem__(self, key)[0]
raise KeyError(key)
def __setitem__(self, key, value):
"""Like :meth:`add` but removes an existing key first.
:param key: the key for the value.
:param value: the value to set.
"""
dict.__setitem__(self, key, [value])
def add(self, key, value):
"""Adds a new value for the key.
.. versionadded:: 0.6
:param key: the key for the value.
:param value: the value to add.
"""
dict.setdefault(self, key, []).append(value)
def getlist(self, key, type=None):
"""Return the list of items for a given key. If that key is not in the
`MultiDict`, the return value will be an empty list. Just as `get`
`getlist` accepts a `type` parameter. All items will be converted
with the callable defined there.
:param key: The key to be looked up.
:param type: A callable that is used to cast the value in the
:class:`MultiDict`. If a :exc:`ValueError` is raised
by this callable the value will be removed from the list.
:return: a :class:`list` of all the values for the key.
"""
try:
rv = dict.__getitem__(self, key)
except KeyError:
return []
if type is None:
return list(rv)
result = []
for item in rv:
try:
result.append(type(item))
except ValueError:
pass
return result
def setlist(self, key, new_list):
"""Remove the old values for a key and add new ones. Note that the list
you pass the values in will be shallow-copied before it is inserted in
the dictionary.
>>> d = MultiDict()
>>> d.setlist('foo', ['1', '2'])
>>> d['foo']
'1'
>>> d.getlist('foo')
['1', '2']
:param key: The key for which the values are set.
:param new_list: An iterable with the new values for the key. Old values
are removed first.
"""
dict.__setitem__(self, key, list(new_list))
def setdefault(self, key, default=None):
"""Returns the value for the key if it is in the dict, otherwise it
returns `default` and sets that value for `key`.
:param key: The key to be looked up.
:param default: The default value to be returned if the key is not
in the dict. If not further specified it's `None`.
"""
if key not in self:
self[key] = default
else:
default = self[key]
return default
def setlistdefault(self, key, default_list=None):
"""Like `setdefault` but sets multiple values. The list returned
is not a copy, but the list that is actually used internally. This
means that you can put new values into the dict by appending items
to the list:
>>> d = MultiDict({"foo": 1})
>>> d.setlistdefault("foo").extend([2, 3])
>>> d.getlist("foo")
[1, 2, 3]
:param key: The key to be looked up.
:param default: An iterable of default values. It is either copied
(in case it was a list) or converted into a list
before returned.
:return: a :class:`list`
"""
if key not in self:
default_list = list(default_list or ())
dict.__setitem__(self, key, default_list)
else:
default_list = dict.__getitem__(self, key)
return default_list
def items(self, multi=False):
"""Return a list of ``(key, value)`` pairs.
:param multi: If set to `True` the list returned will have a
pair for each value of each key. Otherwise it
will only contain pairs for the first value of
each key.
:return: a :class:`list`
"""
return list(self.iteritems(multi))
def lists(self):
"""Return a list of ``(key, values)`` pairs, where values is the list of
all values associated with the key.
:return: a :class:`list`
"""
return list(self.iterlists())
def values(self):
"""Returns a list of the first value on every key's value list.
:return: a :class:`list`.
"""
return [self[key] for key in self.iterkeys()]
def listvalues(self):
"""Return a list of all values associated with a key. Zipping
:meth:`keys` and this is the same as calling :meth:`lists`:
>>> d = MultiDict({"foo": [1, 2, 3]})
>>> zip(d.keys(), d.listvalues()) == d.lists()
True
:return: a :class:`list`
"""
return list(self.iterlistvalues())
def iteritems(self, multi=False):
"""Like :meth:`items` but returns an iterator."""
for key, values in dict.iteritems(self):
if multi:
for value in values:
yield key, value
else:
yield key, values[0]
def iterlists(self):
"""Like :meth:`items` but returns an iterator."""
for key, values in dict.iteritems(self):
yield key, list(values)
def itervalues(self):
"""Like :meth:`values` but returns an iterator."""
for values in dict.itervalues(self):
yield values[0]
def iterlistvalues(self):
"""Like :meth:`listvalues` but returns an iterator."""
return dict.itervalues(self)
def copy(self):
"""Return a shallow copy of this object."""
return self.__class__(self)
def to_dict(self, flat=True):
"""Return the contents as regular dict. If `flat` is `True` the
returned dict will only have the first item present, if `flat` is
`False` all values will be returned as lists.
:param flat: If set to `False` the dict returned will have lists
with all the values in it. Otherwise it will only
contain the first value for each key.
:return: a :class:`dict`
"""
if flat:
return dict(self.iteritems())
return dict(self.lists())
def update(self, other_dict):
"""update() extends rather than replaces existing key lists."""
for key, value in iter_multi_items(other_dict):
MultiDict.add(self, key, value)
def pop(self, key, default=_missing):
"""Pop the first item for a list on the dict. Afterwards the
key is removed from the dict, so additional values are discarded:
>>> d = MultiDict({"foo": [1, 2, 3]})
>>> d.pop("foo")
1
>>> "foo" in d
False
:param key: the key to pop.
:param default: if provided the value to return if the key was
not in the dictionary.
"""
try:
return dict.pop(self, key)[0]
except KeyError as e:
if default is not _missing:
return default
raise KeyError(str(e))
def popitem(self):
"""Pop an item from the dict."""
try:
item = dict.popitem(self)
return (item[0], item[1][0])
except KeyError as e:
raise KeyError(str(e))
def poplist(self, key):
"""Pop the list for a key from the dict. If the key is not in the dict
an empty list is returned.
.. versionchanged:: 0.5
If the key does no longer exist a list is returned instead of
raising an error.
"""
return dict.pop(self, key, [])
def popitemlist(self):
"""Pop a ``(key, list)`` tuple from the dict."""
try:
return dict.popitem(self)
except KeyError as e:
raise KeyError(str(e))
def __copy__(self):
return self.copy()
def __repr__(self):
return '%s(%r)' % (self.__class__.__name__, self.items(multi=True))
libs/requests/packages/urllib3/__init__.py
+1 -1
View File
@@ -10,7 +10,7 @@ urllib3 - Thread-safe connection pooling and re-using.
__author__ = 'Andrey Petrov (andrey.petrov@shazow.net)'
__license__ = 'MIT'
__version__ = '1.3'
__version__ = 'dev'
from .connectionpool import (
libs/requests/packages/urllib3/connectionpool.py
+5 -4
View File
@@ -260,10 +260,11 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
httplib_response = conn.getresponse()
log.debug("\"%s %s %s\" %s %s" %
(method, url,
conn._http_vsn_str, # pylint: disable-msg=W0212
httplib_response.status, httplib_response.length))
# AppEngine doesn't have a version attr.
http_version = getattr(conn, '_http_vsn_str', 'HTTP/?'),
log.debug("\"%s %s %s\" %s %s" % (method, url, http_version,
httplib_response.status,
httplib_response.length))
return httplib_response
libs/requests/safe_mode.py
+38
View File
@@ -0,0 +1,38 @@
# -*- coding: utf-8 -*-
"""
requests.safe_mode
~~~~~~~~~~~~
This module contains a decorator that implements safe_mode.
:copyright: (c) 2012 by Kenneth Reitz.
:license: ISC, see LICENSE for more details.
"""
from .models import Response
from .packages.urllib3.response import HTTPResponse
from .exceptions import RequestException, ConnectionError, HTTPError
import socket
def catch_exceptions_if_in_safe_mode(function):
"""New implementation of safe_mode. We catch all exceptions at the API level
and then return a blank Response object with the error field filled. This decorator
wraps request() in api.py.
"""
def wrapped(method, url, **kwargs):
# if save_mode, we catch exceptions and fill error field
if (kwargs.get('config') and kwargs.get('config').get('safe_mode')) or (kwargs.get('session')
and kwargs.get('session').config.get('safe_mode')):
try:
return function(method, url, **kwargs)
except (RequestException, ConnectionError, HTTPError, socket.timeout) as e:
r = Response()
r.error = e
r.raw = HTTPResponse() # otherwise, tests fail
r.status_code = 0 # with this status_code, content returns None
return r
return function(method, url, **kwargs)
return wrapped
libs/requests/sessions.py
+35 -16
View File
@@ -9,13 +9,15 @@ requests (cookies, auth, proxies).
"""
from copy import deepcopy
from .compat import cookielib
from .cookies import cookiejar_from_dict, remove_cookie_by_name
from .defaults import defaults
from .models import Request
from .hooks import dispatch_hook
from .utils import header_expand
from .packages.urllib3.poolmanager import PoolManager
def merge_kwargs(local_kwarg, default_kwarg):
"""Merges kwarg dictionaries.
@@ -52,7 +54,7 @@ class Session(object):
__attrs__ = [
'headers', 'cookies', 'auth', 'timeout', 'proxies', 'hooks',
'params', 'config', 'verify', 'cert']
'params', 'config', 'verify', 'cert', 'prefetch']
def __init__(self,
@@ -69,7 +71,6 @@ class Session(object):
cert=None):
self.headers = headers or {}
self.cookies = cookies or {}
self.auth = auth
self.timeout = timeout
self.proxies = proxies or {}
@@ -81,16 +82,15 @@ class Session(object):
self.cert = cert
for (k, v) in list(defaults.items()):
self.config.setdefault(k, v)
self.config.setdefault(k, deepcopy(v))
self.init_poolmanager()
# Set up a CookieJar to be used by default
self.cookies = {}
# Add passed cookies in.
if cookies is not None:
self.cookies.update(cookies)
if isinstance(cookies, cookielib.CookieJar):
self.cookies = cookies
else:
self.cookies = cookiejar_from_dict(cookies)
def init_poolmanager(self):
self.poolmanager = PoolManager(
@@ -134,12 +134,12 @@ class Session(object):
:param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`.
:param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`.
:param files: (optional) Dictionary of 'filename': file-like-objects for multipart encoding upload.
:param auth: (optional) Auth tuple to enable Basic/Digest/Custom HTTP Auth.
:param auth: (optional) Auth tuple or callable to enable Basic/Digest/Custom HTTP Auth.
:param timeout: (optional) Float describing the timeout of the request.
:param allow_redirects: (optional) Boolean. Set to True by default.
:param proxies: (optional) Dictionary mapping protocol to the URL of the proxy.
:param return_response: (optional) If False, an un-sent Request object will returned.
:param config: (optional) A configuration dictionary.
:param config: (optional) A configuration dictionary. See ``request.defaults`` for allowed keys and their default values.
:param prefetch: (optional) if ``True``, the response content will be immediately downloaded.
:param verify: (optional) if ``True``, the SSL cert will be verified. A CA_BUNDLE path can also be provided.
:param cert: (optional) if String, path to ssl client cert file (.pem). If Tuple, ('cert', 'key') pair.
@@ -148,7 +148,6 @@ class Session(object):
method = str(method).upper()
# Default empty dicts for dict params.
cookies = {} if cookies is None else cookies
data = {} if data is None else data
files = {} if files is None else files
headers = {} if headers is None else headers
@@ -179,16 +178,39 @@ class Session(object):
allow_redirects=allow_redirects,
proxies=proxies,
config=config,
prefetch=prefetch,
verify=verify,
cert=cert,
_poolmanager=self.poolmanager
)
# merge session cookies into passed-in ones
dead_cookies = None
# passed-in cookies must become a CookieJar:
if not isinstance(cookies, cookielib.CookieJar):
args['cookies'] = cookiejar_from_dict(cookies)
# support unsetting cookies that have been passed in with None values
# this is only meaningful when `cookies` is a dict ---
# for a real CookieJar, the client should use session.cookies.clear()
if cookies is not None:
dead_cookies = [name for name in cookies if cookies[name] is None]
# merge the session's cookies into the passed-in cookies:
for cookie in self.cookies:
args['cookies'].set_cookie(cookie)
# remove the unset cookies from the jar we'll be using with the current request
# (but not from the session's own store of cookies):
if dead_cookies is not None:
for name in dead_cookies:
remove_cookie_by_name(args['cookies'], name)
# Merge local kwargs with session kwargs.
for attr in self.__attrs__:
# we already merged cookies:
if attr == 'cookies':
continue
session_val = getattr(self, attr, None)
local_val = args.get(attr)
args[attr] = merge_kwargs(local_val, session_val)
# Arguments manipulation hook.
@@ -207,9 +229,6 @@ class Session(object):
# Send the HTTP Request.
r.send(prefetch=prefetch)
# Send any cookies back up the to the session.
self.cookies.update(r.response.cookies)
# Return the response.
return r.response
libs/requests/structures.py
+1 -1
View File
@@ -30,7 +30,7 @@ class CaseInsensitiveDict(dict):
self._clear_lower_keys()
def __delitem__(self, key):
dict.__delitem__(self, key)
dict.__delitem__(self, self.lower_keys.get(key.lower(), key))
self._lower_keys.clear()
def __contains__(self, key):
libs/requests/utils.py
+92 -108
View File
@@ -12,18 +12,46 @@ that are also useful for external consumption.
import cgi
import codecs
import os
import random
import re
import zlib
from netrc import netrc, NetrcParseError
from .compat import parse_http_list as _parse_list_header
from .compat import quote, cookielib, SimpleCookie, is_py2, urlparse
from .compat import basestring, bytes, str
from .compat import quote, urlparse, basestring, bytes, str
from .cookies import RequestsCookieJar, cookiejar_from_dict
_hush_pyflakes = (RequestsCookieJar,)
CERTIFI_BUNDLE_PATH = None
try:
# see if requests's own CA certificate bundle is installed
import certifi
CERTIFI_BUNDLE_PATH = certifi.where()
except ImportError:
pass
NETRC_FILES = ('.netrc', '_netrc')
# common paths for the OS's CA certificate bundle
POSSIBLE_CA_BUNDLE_PATHS = [
# Red Hat, CentOS, Fedora and friends (provided by the ca-certificates package):
'/etc/pki/tls/certs/ca-bundle.crt',
# Ubuntu, Debian, and friends (provided by the ca-certificates package):
'/etc/ssl/certs/ca-certificates.crt',
# FreeBSD (provided by the ca_root_nss package):
'/usr/local/share/certs/ca-root-nss.crt',
]
def get_os_ca_bundle_path():
"""Try to pick an available CA certificate bundle provided by the OS."""
for path in POSSIBLE_CA_BUNDLE_PATHS:
if os.path.exists(path):
return path
return None
# if certifi is installed, use its CA bundle;
# otherwise, try and use the OS bundle
DEFAULT_CA_BUNDLE_PATH = CERTIFI_BUNDLE_PATH or get_os_ca_bundle_path()
def dict_to_sequence(d):
"""Returns an internal sequence dictionary update."""
@@ -37,52 +65,39 @@ def dict_to_sequence(d):
def get_netrc_auth(url):
"""Returns the Requests tuple auth for a given url from netrc."""
locations = (os.path.expanduser('~/{0}'.format(f)) for f in NETRC_FILES)
netrc_path = None
for loc in locations:
if os.path.exists(loc) and not netrc_path:
netrc_path = loc
# Abort early if there isn't one.
if netrc_path is None:
return netrc_path
ri = urlparse(url)
# Strip port numbers from netloc
host = ri.netloc.split(':')[0]
try:
_netrc = netrc(netrc_path).authenticators(host)
if _netrc:
# Return with login / password
login_i = (0 if _netrc[0] else 1)
return (_netrc[login_i], _netrc[2])
except (NetrcParseError, IOError, AttributeError):
# If there was a parsing error or a permissions issue reading the file,
# we'll just skip netrc auth
locations = (os.path.expanduser('~/{0}'.format(f)) for f in NETRC_FILES)
netrc_path = None
for loc in locations:
if os.path.exists(loc) and not netrc_path:
netrc_path = loc
# Abort early if there isn't one.
if netrc_path is None:
return netrc_path
ri = urlparse(url)
# Strip port numbers from netloc
host = ri.netloc.split(':')[0]
try:
_netrc = netrc(netrc_path).authenticators(host)
if _netrc:
# Return with login / password
login_i = (0 if _netrc[0] else 1)
return (_netrc[login_i], _netrc[2])
except (NetrcParseError, IOError):
# If there was a parsing error or a permissions issue reading the file,
# we'll just skip netrc auth
pass
# AppEngine hackiness.
except AttributeError:
pass
def dict_from_string(s):
"""Returns a MultiDict with Cookies."""
cookies = dict()
try:
c = SimpleCookie()
c.load(s)
for k, v in list(c.items()):
cookies.update({k: v.value})
# This stuff is not to be trusted.
except Exception:
pass
return cookies
def guess_filename(obj):
"""Tries to guess the filename of the given object."""
name = getattr(obj, 'name', None)
@@ -231,15 +246,6 @@ def header_expand(headers):
return ''.join(collector)
def randombytes(n):
"""Return n random bytes."""
if is_py2:
L = [chr(random.randrange(0, 256)) for i in range(n)]
else:
L = [chr(random.randrange(0, 256)).encode('utf-8') for i in range(n)]
return b"".join(L)
def dict_from_cookiejar(cj):
"""Returns a key/value dictionary from a CookieJar.
@@ -257,24 +263,6 @@ def dict_from_cookiejar(cj):
return cookie_dict
def cookiejar_from_dict(cookie_dict):
"""Returns a CookieJar from a key/value dictionary.
:param cookie_dict: Dict of key/values to insert into CookieJar.
"""
# return cookiejar if one was passed in
if isinstance(cookie_dict, cookielib.CookieJar):
return cookie_dict
# create cookiejar
cj = cookielib.CookieJar()
cj = add_dict_to_cookiejar(cj, cookie_dict)
return cj
def add_dict_to_cookiejar(cj, cookie_dict):
"""Returns a CookieJar from a key/value dictionary.
@@ -282,31 +270,9 @@ def add_dict_to_cookiejar(cj, cookie_dict):
:param cookie_dict: Dict of key/values to insert into CookieJar.
"""
for k, v in list(cookie_dict.items()):
cookie = cookielib.Cookie(
version=0,
name=k,
value=v,
port=None,
port_specified=False,
domain='',
domain_specified=False,
domain_initial_dot=False,
path='/',
path_specified=True,
secure=False,
expires=None,
discard=True,
comment=None,
comment_url=None,
rest={'HttpOnly': None},
rfc2109=False
)
# add cookie to cookiejar
cj2 = cookiejar_from_dict(cookie_dict)
for cookie in cj2:
cj.set_cookie(cookie)
return cj
@@ -442,21 +408,23 @@ UNRESERVED_SET = frozenset(
def unquote_unreserved(uri):
"""Un-escape any percent-escape sequences in a URI that are unreserved
characters.
This leaves all reserved, illegal and non-ASCII bytes encoded.
characters. This leaves all reserved, illegal and non-ASCII bytes encoded.
"""
parts = uri.split('%')
for i in range(1, len(parts)):
h = parts[i][0:2]
if len(h) == 2:
c = chr(int(h, 16))
if c in UNRESERVED_SET:
parts[i] = c + parts[i][2:]
try:
parts = uri.split('%')
for i in range(1, len(parts)):
h = parts[i][0:2]
if len(h) == 2 and h.isalnum():
c = chr(int(h, 16))
if c in UNRESERVED_SET:
parts[i] = c + parts[i][2:]
else:
parts[i] = '%' + parts[i]
else:
parts[i] = '%' + parts[i]
else:
parts[i] = '%' + parts[i]
return ''.join(parts)
return ''.join(parts)
except ValueError:
return uri
def requote_uri(uri):
@@ -469,3 +437,19 @@ def requote_uri(uri):
# Then quote only illegal characters (do not quote reserved, unreserved,
# or '%')
return quote(unquote_unreserved(uri), safe="!#$%&'()*+,/:;=?@[]~")
def get_environ_proxies():
"""Return a dict of environment proxies."""
proxy_keys = [
'all',
'http',
'https',
'ftp',
'socks',
'no'
]
get_proxy = lambda k: os.environ.get(k) or os.environ.get(k.upper())
proxies = [(key, get_proxy(key + '_proxy')) for key in proxy_keys]
return dict([(key, val) for (key, val) in proxies if val])