dinislage/CouchPotatoServer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update Tornado lib

Browse Source
This commit is contained in:
Ruud
2012-12-03 13:31:34 +01:00
parent f0a3358561
commit 802338a934
33 changed files with 2510 additions and 1336 deletions
Download Patch File Download Diff File Expand all files Collapse all files
libs/tornado/simple_httpclient.py
+119 -103
View File
@@ -2,9 +2,11 @@
from __future__ import absolute_import, division, with_statement
from tornado.escape import utf8, _unicode, native_str
from tornado.httpclient import HTTPRequest, HTTPResponse, HTTPError, AsyncHTTPClient, main
from tornado.httpclient import HTTPRequest, HTTPResponse, HTTPError, AsyncHTTPClient, main, _RequestProxy
from tornado.httputil import HTTPHeaders
from tornado.iostream import IOStream, SSLIOStream
from tornado.netutil import Resolver
from tornado.log import gen_log
from tornado import stack_context
from tornado.util import b, GzipDecompressor
@@ -13,7 +15,6 @@ import collections
import contextlib
import copy
import functools
import logging
import os.path
import re
import socket
@@ -39,17 +40,7 @@ class SimpleAsyncHTTPClient(AsyncHTTPClient):
This class implements an HTTP 1.1 client on top of Tornado's IOStreams.
It does not currently implement all applicable parts of the HTTP
specification, but it does enough to work with major web service APIs
(mostly tested against the Twitter API so far).
This class has not been tested extensively in production and
should be considered somewhat experimental as of the release of
tornado 1.2. It is intended to become the default AsyncHTTPClient
implementation in a future release. It may either be used
directly, or to facilitate testing of this class with an existing
application, setting the environment variable
USE_SIMPLE_HTTPCLIENT=1 will cause this class to transparently
replace tornado.httpclient.AsyncHTTPClient.
specification, but it does enough to work with major web service APIs.
Some features found in the curl-based AsyncHTTPClient are not yet
supported. In particular, proxies are not supported, connections
@@ -61,19 +52,18 @@ class SimpleAsyncHTTPClient(AsyncHTTPClient):
"""
def initialize(self, io_loop=None, max_clients=10,
max_simultaneous_connections=None,
hostname_mapping=None, max_buffer_size=104857600):
hostname_mapping=None, max_buffer_size=104857600,
resolver=None, defaults=None):
"""Creates a AsyncHTTPClient.
Only a single AsyncHTTPClient instance exists per IOLoop
in order to provide limitations on the number of pending connections.
force_instance=True may be used to suppress this behavior.
max_clients is the number of concurrent requests that can be in
progress. max_simultaneous_connections has no effect and is accepted
only for compatibility with the curl-based AsyncHTTPClient. Note
that these arguments are only used when the client is first created,
and will be ignored when an existing client is reused.
max_clients is the number of concurrent requests that can be
in progress. Note that this arguments are only used when the
client is first created, and will be ignored when an existing
client is reused.
hostname_mapping is a dictionary mapping hostnames to IP addresses.
It can be used to make local DNS changes when modifying system-wide
@@ -89,6 +79,10 @@ class SimpleAsyncHTTPClient(AsyncHTTPClient):
self.active = {}
self.hostname_mapping = hostname_mapping
self.max_buffer_size = max_buffer_size
self.resolver = resolver or Resolver(io_loop=io_loop)
self.defaults = dict(HTTPRequest._DEFAULTS)
if defaults is not None:
self.defaults.update(defaults)
def fetch(self, request, callback, **kwargs):
if not isinstance(request, HTTPRequest):
@@ -97,11 +91,12 @@ class SimpleAsyncHTTPClient(AsyncHTTPClient):
# so make sure we don't modify the caller's object. This is also
# where normal dicts get converted to HTTPHeaders objects.
request.headers = HTTPHeaders(request.headers)
request = _RequestProxy(request, self.defaults)
callback = stack_context.wrap(callback)
self.queue.append((request, callback))
self._process_queue()
if self.queue:
logging.debug("max_clients limit reached, request queued. "
gen_log.debug("max_clients limit reached, request queued. "
"%d active, %d queued requests." % (
len(self.active), len(self.queue)))
@@ -126,12 +121,13 @@ class _HTTPConnection(object):
def __init__(self, io_loop, client, request, release_callback,
final_callback, max_buffer_size):
self.start_time = time.time()
self.start_time = io_loop.time()
self.io_loop = io_loop
self.client = client
self.request = request
self.release_callback = release_callback
self.final_callback = final_callback
self.max_buffer_size = max_buffer_size
self.code = None
self.headers = None
self.chunks = None
@@ -139,16 +135,16 @@ class _HTTPConnection(object):
# Timeout handle returned by IOLoop.add_timeout
self._timeout = None
with stack_context.StackContext(self.cleanup):
parsed = urlparse.urlsplit(_unicode(self.request.url))
if ssl is None and parsed.scheme == "https":
self.parsed = urlparse.urlsplit(_unicode(self.request.url))
if ssl is None and self.parsed.scheme == "https":
raise ValueError("HTTPS requires either python2.6+ or "
"curl_httpclient")
if parsed.scheme not in ("http", "https"):
if self.parsed.scheme not in ("http", "https"):
raise ValueError("Unsupported url scheme: %s" %
self.request.url)
# urlsplit results have hostname and port results, but they
# didn't support ipv6 literals until python 2.7.
netloc = parsed.netloc
netloc = self.parsed.netloc
if "@" in netloc:
userpass, _, netloc = netloc.rpartition("@")
match = re.match(r'^(.+):(\d+)$', netloc)
@@ -157,11 +153,11 @@ class _HTTPConnection(object):
port = int(match.group(2))
else:
host = netloc
port = 443 if parsed.scheme == "https" else 80
port = 443 if self.parsed.scheme == "https" else 80
if re.match(r'^\[.*\]$', host):
# raw ipv6 addresses in urls are enclosed in brackets
host = host[1:-1]
parsed_hostname = host # save final parsed host for _on_connect
self.parsed_hostname = host # save final host for _on_connect
if self.client.hostname_mapping is not None:
host = self.client.hostname_mapping.get(host, host)
@@ -172,66 +168,67 @@ class _HTTPConnection(object):
# so restrict to ipv4 by default.
af = socket.AF_INET
addrinfo = socket.getaddrinfo(host, port, af, socket.SOCK_STREAM,
0, 0)
af, socktype, proto, canonname, sockaddr = addrinfo[0]
self.client.resolver.getaddrinfo(
host, port, af, socket.SOCK_STREAM, 0, 0,
callback=self._on_resolve)
if parsed.scheme == "https":
ssl_options = {}
if request.validate_cert:
ssl_options["cert_reqs"] = ssl.CERT_REQUIRED
if request.ca_certs is not None:
ssl_options["ca_certs"] = request.ca_certs
else:
ssl_options["ca_certs"] = _DEFAULT_CA_CERTS
if request.client_key is not None:
ssl_options["keyfile"] = request.client_key
if request.client_cert is not None:
ssl_options["certfile"] = request.client_cert
def _on_resolve(self, future):
af, socktype, proto, canonname, sockaddr = future.result()[0]
# SSL interoperability is tricky. We want to disable
# SSLv2 for security reasons; it wasn't disabled by default
# until openssl 1.0. The best way to do this is to use
# the SSL_OP_NO_SSLv2, but that wasn't exposed to python
# until 3.2. Python 2.7 adds the ciphers argument, which
# can also be used to disable SSLv2. As a last resort
# on python 2.6, we set ssl_version to SSLv3. This is
# more narrow than we'd like since it also breaks
# compatibility with servers configured for TLSv1 only,
# but nearly all servers support SSLv3:
# http://blog.ivanristic.com/2011/09/ssl-survey-protocol-support.html
if sys.version_info >= (2, 7):
ssl_options["ciphers"] = "DEFAULT:!SSLv2"
else:
# This is really only necessary for pre-1.0 versions
# of openssl, but python 2.6 doesn't expose version
# information.
ssl_options["ssl_version"] = ssl.PROTOCOL_SSLv3
self.stream = SSLIOStream(socket.socket(af, socktype, proto),
io_loop=self.io_loop,
ssl_options=ssl_options,
max_buffer_size=max_buffer_size)
if self.parsed.scheme == "https":
ssl_options = {}
if self.request.validate_cert:
ssl_options["cert_reqs"] = ssl.CERT_REQUIRED
if self.request.ca_certs is not None:
ssl_options["ca_certs"] = self.request.ca_certs
else:
self.stream = IOStream(socket.socket(af, socktype, proto),
io_loop=self.io_loop,
max_buffer_size=max_buffer_size)
timeout = min(request.connect_timeout, request.request_timeout)
if timeout:
self._timeout = self.io_loop.add_timeout(
self.start_time + timeout,
stack_context.wrap(self._on_timeout))
self.stream.set_close_callback(self._on_close)
self.stream.connect(sockaddr,
functools.partial(self._on_connect, parsed,
parsed_hostname))
ssl_options["ca_certs"] = _DEFAULT_CA_CERTS
if self.request.client_key is not None:
ssl_options["keyfile"] = self.request.client_key
if self.request.client_cert is not None:
ssl_options["certfile"] = self.request.client_cert
# SSL interoperability is tricky. We want to disable
# SSLv2 for security reasons; it wasn't disabled by default
# until openssl 1.0. The best way to do this is to use
# the SSL_OP_NO_SSLv2, but that wasn't exposed to python
# until 3.2. Python 2.7 adds the ciphers argument, which
# can also be used to disable SSLv2. As a last resort
# on python 2.6, we set ssl_version to SSLv3. This is
# more narrow than we'd like since it also breaks
# compatibility with servers configured for TLSv1 only,
# but nearly all servers support SSLv3:
# http://blog.ivanristic.com/2011/09/ssl-survey-protocol-support.html
if sys.version_info >= (2, 7):
ssl_options["ciphers"] = "DEFAULT:!SSLv2"
else:
# This is really only necessary for pre-1.0 versions
# of openssl, but python 2.6 doesn't expose version
# information.
ssl_options["ssl_version"] = ssl.PROTOCOL_SSLv3
self.stream = SSLIOStream(socket.socket(af, socktype, proto),
io_loop=self.io_loop,
ssl_options=ssl_options,
max_buffer_size=self.max_buffer_size)
else:
self.stream = IOStream(socket.socket(af, socktype, proto),
io_loop=self.io_loop,
max_buffer_size=self.max_buffer_size)
timeout = min(self.request.connect_timeout, self.request.request_timeout)
if timeout:
self._timeout = self.io_loop.add_timeout(
self.start_time + timeout,
stack_context.wrap(self._on_timeout))
self.stream.set_close_callback(self._on_close)
self.stream.connect(sockaddr, self._on_connect)
def _on_timeout(self):
self._timeout = None
if self.final_callback is not None:
raise HTTPError(599, "Timeout")
def _on_connect(self, parsed, parsed_hostname):
def _on_connect(self):
if self._timeout is not None:
self.io_loop.remove_timeout(self._timeout)
self._timeout = None
@@ -243,10 +240,10 @@ class _HTTPConnection(object):
isinstance(self.stream, SSLIOStream)):
match_hostname(self.stream.socket.getpeercert(),
# ipv6 addresses are broken (in
# parsed.hostname) until 2.7, here is
# self.parsed.hostname) until 2.7, here is
# correctly parsed value calculated in
# __init__
parsed_hostname)
self.parsed_hostname)
if (self.request.method not in self._SUPPORTED_METHODS and
not self.request.allow_nonstandard_methods):
raise KeyError("unknown method %s" % self.request.method)
@@ -258,13 +255,13 @@ class _HTTPConnection(object):
if "Connection" not in self.request.headers:
self.request.headers["Connection"] = "close"
if "Host" not in self.request.headers:
if '@' in parsed.netloc:
self.request.headers["Host"] = parsed.netloc.rpartition('@')[-1]
if '@' in self.parsed.netloc:
self.request.headers["Host"] = self.parsed.netloc.rpartition('@')[-1]
else:
self.request.headers["Host"] = parsed.netloc
self.request.headers["Host"] = self.parsed.netloc
username, password = None, None
if parsed.username is not None:
username, password = parsed.username, parsed.password
if self.parsed.username is not None:
username, password = self.parsed.username, self.parsed.password
elif self.request.auth_username is not None:
username = self.request.auth_username
password = self.request.auth_password or ''
@@ -287,8 +284,8 @@ class _HTTPConnection(object):
self.request.headers["Content-Type"] = "application/x-www-form-urlencoded"
if self.request.use_gzip:
self.request.headers["Accept-Encoding"] = "gzip"
req_path = ((parsed.path or '/') +
(('?' + parsed.query) if parsed.query else ''))
req_path = ((self.parsed.path or '/') +
(('?' + self.parsed.query) if self.parsed.query else ''))
request_lines = [utf8("%s %s HTTP/1.1" % (self.request.method,
req_path))]
for k, v in self.request.headers.get_all():
@@ -319,23 +316,32 @@ class _HTTPConnection(object):
try:
yield
except Exception, e:
logging.warning("uncaught exception", exc_info=True)
gen_log.warning("uncaught exception", exc_info=True)
self._run_callback(HTTPResponse(self.request, 599, error=e,
request_time=time.time() - self.start_time,
request_time=self.io_loop.time() - self.start_time,
))
if hasattr(self, "stream"):
self.stream.close()
def _on_close(self):
if self.final_callback is not None:
raise HTTPError(599, "Connection closed")
message = "Connection closed"
if self.stream.error:
message = str(self.stream.error)
raise HTTPError(599, message)
def _on_headers(self, data):
data = native_str(data.decode("latin1"))
first_line, _, header_data = data.partition("\n")
match = re.match("HTTP/1.[01] ([0-9]+)", first_line)
match = re.match("HTTP/1.[01] ([0-9]+) ([^\r]*)", first_line)
assert match
self.code = int(match.group(1))
code = int(match.group(1))
if 100 <= code < 200:
self.stream.read_until_regex(b("\r?\n\r?\n"), self._on_headers)
return
else:
self.code = code
self.reason = match.group(2)
self.headers = HTTPHeaders.parse(header_data)
if "Content-Length" in self.headers:
@@ -353,15 +359,18 @@ class _HTTPConnection(object):
content_length = None
if self.request.header_callback is not None:
# re-attach the newline we split on earlier
self.request.header_callback(first_line + _)
for k, v in self.headers.get_all():
self.request.header_callback("%s: %s\r\n" % (k, v))
self.request.header_callback('\r\n')
if self.request.method == "HEAD":
# HEAD requests never have content, even though they may have
# content-length headers
if self.request.method == "HEAD" or self.code == 304:
# HEAD requests and 304 responses never have content, even
# though they may have content-length headers
self._on_body(b(""))
return
if 100 <= self.code < 200 or self.code in (204, 304):
if 100 <= self.code < 200 or self.code == 204:
# These response codes never have bodies
# http://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.3
if ("Transfer-Encoding" in self.headers or
@@ -391,14 +400,20 @@ class _HTTPConnection(object):
if (self.request.follow_redirects and
self.request.max_redirects > 0 and
self.code in (301, 302, 303, 307)):
new_request = copy.copy(self.request)
assert isinstance(self.request, _RequestProxy)
new_request = copy.copy(self.request.request)
new_request.url = urlparse.urljoin(self.request.url,
self.headers["Location"])
new_request.max_redirects -= 1
new_request.max_redirects = self.request.max_redirects - 1
del new_request.headers["Host"]
# http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3.4
# client SHOULD make a GET request
if self.code == 303:
# Client SHOULD make a GET request after a 303.
# According to the spec, 302 should be followed by the same
# method as the original request, but in practice browsers
# treat 302 the same as 303, and many servers use 302 for
# compatibility with pre-HTTP/1.1 user agents which don't
# understand the 303 status.
if self.code in (302, 303):
new_request.method = "GET"
new_request.body = None
for h in ["Content-Length", "Content-Type",
@@ -426,8 +441,9 @@ class _HTTPConnection(object):
else:
buffer = BytesIO(data) # TODO: don't require one big string?
response = HTTPResponse(original_request,
self.code, headers=self.headers,
request_time=time.time() - self.start_time,
self.code, reason=self.reason,
headers=self.headers,
request_time=self.io_loop.time() - self.start_time,
buffer=buffer,
effective_url=self.request.url)
self._run_callback(response)