dinislage/CouchPotatoServer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update to Tornado 3.0

Browse Source
This commit is contained in:
Ruud
2013-04-19 14:49:27 +02:00
parent 4f6b31d14a
commit 0e90739786
29 changed files with 1932 additions and 1166 deletions
Download Patch File Download Diff File Expand all files Collapse all files
libs/tornado/httpclient.py
+90 -87
View File
@@ -1,36 +1,35 @@
"""Blocking and non-blocking HTTP client interfaces.
This module defines a common interface shared by two implementations,
`simple_httpclient` and `curl_httpclient`. Applications may either
``simple_httpclient`` and ``curl_httpclient``. Applications may either
instantiate their chosen implementation class directly or use the
`AsyncHTTPClient` class from this module, which selects an implementation
that can be overridden with the `AsyncHTTPClient.configure` method.
The default implementation is `simple_httpclient`, and this is expected
The default implementation is ``simple_httpclient``, and this is expected
to be suitable for most users' needs. However, some applications may wish
to switch to `curl_httpclient` for reasons such as the following:
to switch to ``curl_httpclient`` for reasons such as the following:
* `curl_httpclient` has some features not found in `simple_httpclient`,
* ``curl_httpclient`` has some features not found in ``simple_httpclient``,
including support for HTTP proxies and the ability to use a specified
network interface.
* `curl_httpclient` is more likely to be compatible with sites that are
* ``curl_httpclient`` is more likely to be compatible with sites that are
not-quite-compliant with the HTTP spec, or sites that use little-exercised
features of HTTP.
* `simple_httpclient` only supports SSL on Python 2.6 and above.
* ``curl_httpclient`` is faster.
* `curl_httpclient` is faster
* ``curl_httpclient`` was the default prior to Tornado 2.0.
* `curl_httpclient` was the default prior to Tornado 2.0.
Note that if you are using `curl_httpclient`, it is highly recommended that
Note that if you are using ``curl_httpclient``, it is highly recommended that
you use a recent version of ``libcurl`` and ``pycurl``. Currently the minimum
supported version is 7.18.2, and the recommended version is 7.21.1 or newer.
"""
from __future__ import absolute_import, division, print_function, with_statement
import functools
import time
import weakref
@@ -52,15 +51,15 @@ class HTTPClient(object):
try:
response = http_client.fetch("http://www.google.com/")
print response.body
except httpclient.HTTPError, e:
except httpclient.HTTPError as e:
print "Error:", e
httpclient.close()
"""
def __init__(self, async_client_class=None, **kwargs):
self._io_loop = IOLoop()
if async_client_class is None:
async_client_class = AsyncHTTPClient
self._async_client = async_client_class(self._io_loop, **kwargs)
self._response = None
self._closed = False
def __del__(self):
@@ -82,13 +81,8 @@ class HTTPClient(object):
If an error occurs during the fetch, we raise an `HTTPError`.
"""
def callback(response):
self._response = response
self._io_loop.stop()
self._async_client.fetch(request, callback, **kwargs)
self._io_loop.start()
response = self._response
self._response = None
response = self._io_loop.run_sync(functools.partial(
self._async_client.fetch, request, **kwargs))
response.rethrow()
return response
@@ -98,26 +92,23 @@ class AsyncHTTPClient(Configurable):
Example usage::
import ioloop
def handle_request(response):
if response.error:
print "Error:", response.error
else:
print response.body
ioloop.IOLoop.instance().stop()
http_client = httpclient.AsyncHTTPClient()
http_client = AsyncHTTPClient()
http_client.fetch("http://www.google.com/", handle_request)
ioloop.IOLoop.instance().start()
The constructor for this class is magic in several respects: It actually
creates an instance of an implementation-specific subclass, and instances
are reused as a kind of pseudo-singleton (one per IOLoop). The keyword
argument force_instance=True can be used to suppress this singleton
behavior. Constructor arguments other than io_loop and force_instance
are deprecated. The implementation subclass as well as arguments to
its constructor can be set with the static method configure()
The constructor for this class is magic in several respects: It
actually creates an instance of an implementation-specific
subclass, and instances are reused as a kind of pseudo-singleton
(one per `.IOLoop`). The keyword argument ``force_instance=True``
can be used to suppress this singleton behavior. Constructor
arguments other than ``io_loop`` and ``force_instance`` are
deprecated. The implementation subclass as well as arguments to
its constructor can be set with the static method `configure()`
"""
@classmethod
def configurable_base(cls):
@@ -136,7 +127,7 @@ class AsyncHTTPClient(Configurable):
return getattr(cls, attr_name)
def __new__(cls, io_loop=None, force_instance=False, **kwargs):
io_loop = io_loop or IOLoop.instance()
io_loop = io_loop or IOLoop.current()
if io_loop in cls._async_clients() and not force_instance:
return cls._async_clients()[io_loop]
instance = super(AsyncHTTPClient, cls).__new__(cls, io_loop=io_loop,
@@ -152,25 +143,29 @@ class AsyncHTTPClient(Configurable):
self.defaults.update(defaults)
def close(self):
"""Destroys this http client, freeing any file descriptors used.
"""Destroys this HTTP client, freeing any file descriptors used.
Not needed in normal use, but may be helpful in unittests that
create and destroy http clients. No other methods may be called
on the AsyncHTTPClient after close().
on the `AsyncHTTPClient` after ``close()``.
"""
if self._async_clients().get(self.io_loop) is self:
del self._async_clients()[self.io_loop]
def fetch(self, request, callback=None, **kwargs):
"""Executes a request, calling callback with an `HTTPResponse`.
"""Executes a request, asynchronously returning an `HTTPResponse`.
The request may be either a string URL or an `HTTPRequest` object.
If it is a string, we construct an `HTTPRequest` using any additional
kwargs: ``HTTPRequest(request, **kwargs)``
If an error occurs during the fetch, the HTTPResponse given to the
callback has a non-None error attribute that contains the exception
encountered during the request. You can call response.rethrow() to
throw the exception (if any) in the callback.
This method returns a `.Future` whose result is an
`HTTPResponse`. The ``Future`` wil raise an `HTTPError` if
the request returned a non-200 response code.
If a ``callback`` is given, it will be invoked with the `HTTPResponse`.
In the callback interface, `HTTPError` is not automatically raised.
Instead, you must check the response's ``error`` attribute or
call its `~HTTPResponse.rethrow` method.
"""
if not isinstance(request, HTTPRequest):
request = HTTPRequest(url=request, **kwargs)
@@ -182,6 +177,7 @@ class AsyncHTTPClient(Configurable):
future = Future()
if callback is not None:
callback = stack_context.wrap(callback)
def handle_future(future):
exc = future.exception()
if isinstance(exc, HTTPError) and exc.response is not None:
@@ -194,6 +190,7 @@ class AsyncHTTPClient(Configurable):
response = future.result()
self.io_loop.add_callback(callback, response)
future.add_done_callback(handle_future)
def handle_response(response):
if response.error:
future.set_exception(response.error)
@@ -207,19 +204,19 @@ class AsyncHTTPClient(Configurable):
@classmethod
def configure(cls, impl, **kwargs):
"""Configures the AsyncHTTPClient subclass to use.
"""Configures the `AsyncHTTPClient` subclass to use.
AsyncHTTPClient() actually creates an instance of a subclass.
``AsyncHTTPClient()`` actually creates an instance of a subclass.
This method may be called with either a class object or the
fully-qualified name of such a class (or None to use the default,
SimpleAsyncHTTPClient)
fully-qualified name of such a class (or ``None`` to use the default,
``SimpleAsyncHTTPClient``)
If additional keyword arguments are given, they will be passed
to the constructor of each subclass instance created. The
keyword argument max_clients determines the maximum number of
simultaneous fetch() operations that can execute in parallel
on each IOLoop. Additional arguments may be supported depending
on the implementation class in use.
keyword argument ``max_clients`` determines the maximum number
of simultaneous `~AsyncHTTPClient.fetch()` operations that can
execute in parallel on each `.IOLoop`. Additional arguments
may be supported depending on the implementation class in use.
Example::
@@ -245,7 +242,7 @@ class HTTPRequest(object):
validate_cert=True)
def __init__(self, url, method="GET", headers=None, body=None,
auth_username=None, auth_password=None,
auth_username=None, auth_password=None, auth_mode=None,
connect_timeout=None, request_timeout=None,
if_modified_since=None, follow_redirects=None,
max_redirects=None, user_agent=None, use_gzip=None,
@@ -256,59 +253,61 @@ class HTTPRequest(object):
validate_cert=None, ca_certs=None,
allow_ipv6=None,
client_key=None, client_cert=None):
r"""Creates an `HTTPRequest`.
All parameters except `url` are optional.
r"""All parameters except ``url`` are optional.
:arg string url: URL to fetch
:arg string method: HTTP method, e.g. "GET" or "POST"
:arg headers: Additional HTTP headers to pass on the request
:type headers: `~tornado.httputil.HTTPHeaders` or `dict`
:arg string auth_username: Username for HTTP "Basic" authentication
:arg string auth_password: Password for HTTP "Basic" authentication
:arg string auth_username: Username for HTTP authentication
:arg string auth_password: Password for HTTP authentication
:arg string auth_mode: Authentication mode; default is "basic".
Allowed values are implementation-defined; ``curl_httpclient``
supports "basic" and "digest"; ``simple_httpclient`` only supports
"basic"
:arg float connect_timeout: Timeout for initial connection in seconds
:arg float request_timeout: Timeout for entire request in seconds
:arg datetime if_modified_since: Timestamp for ``If-Modified-Since``
header
:arg if_modified_since: Timestamp for ``If-Modified-Since`` header
:type if_modified_since: `datetime` or `float`
:arg bool follow_redirects: Should redirects be followed automatically
or return the 3xx response?
:arg int max_redirects: Limit for `follow_redirects`
:arg int max_redirects: Limit for ``follow_redirects``
:arg string user_agent: String to send as ``User-Agent`` header
:arg bool use_gzip: Request gzip encoding from the server
:arg string network_interface: Network interface to use for request
:arg callable streaming_callback: If set, `streaming_callback` will
:arg callable streaming_callback: If set, ``streaming_callback`` will
be run with each chunk of data as it is received, and
`~HTTPResponse.body` and `~HTTPResponse.buffer` will be empty in
``HTTPResponse.body`` and ``HTTPResponse.buffer`` will be empty in
the final response.
:arg callable header_callback: If set, `header_callback` will
:arg callable header_callback: If set, ``header_callback`` will
be run with each header line as it is received (including the
first line, e.g. ``HTTP/1.0 200 OK\r\n``, and a final line
containing only ``\r\n``. All lines include the trailing newline
characters). `~HTTPResponse.headers` will be empty in the final
characters). ``HTTPResponse.headers`` will be empty in the final
response. This is most useful in conjunction with
`streaming_callback`, because it's the only way to get access to
``streaming_callback``, because it's the only way to get access to
header data while the request is in progress.
:arg callable prepare_curl_callback: If set, will be called with
a `pycurl.Curl` object to allow the application to make additional
`setopt` calls.
a ``pycurl.Curl`` object to allow the application to make additional
``setopt`` calls.
:arg string proxy_host: HTTP proxy hostname. To use proxies,
`proxy_host` and `proxy_port` must be set; `proxy_username` and
`proxy_pass` are optional. Proxies are currently only support
with `curl_httpclient`.
``proxy_host`` and ``proxy_port`` must be set; ``proxy_username`` and
``proxy_pass`` are optional. Proxies are currently only supported
with ``curl_httpclient``.
:arg int proxy_port: HTTP proxy port
:arg string proxy_username: HTTP proxy username
:arg string proxy_password: HTTP proxy password
:arg bool allow_nonstandard_methods: Allow unknown values for `method`
:arg bool allow_nonstandard_methods: Allow unknown values for ``method``
argument?
:arg bool validate_cert: For HTTPS requests, validate the server's
certificate?
:arg string ca_certs: filename of CA certificates in PEM format,
or None to use defaults. Note that in `curl_httpclient`, if
any request uses a custom `ca_certs` file, they all must (they
don't have to all use the same `ca_certs`, but it's not possible
to mix requests with ca_certs and requests that use the defaults.
or None to use defaults. Note that in ``curl_httpclient``, if
any request uses a custom ``ca_certs`` file, they all must (they
don't have to all use the same ``ca_certs``, but it's not possible
to mix requests with ``ca_certs`` and requests that use the defaults.
:arg bool allow_ipv6: Use IPv6 when available? Default is false in
`simple_httpclient` and true in `curl_httpclient`
``simple_httpclient`` and true in ``curl_httpclient``
:arg string client_key: Filename for client SSL key, if any
:arg string client_cert: Filename for client SSL certificate, if any
"""
@@ -327,6 +326,7 @@ class HTTPRequest(object):
self.body = utf8(body)
self.auth_username = auth_username
self.auth_password = auth_password
self.auth_mode = auth_mode
self.connect_timeout = connect_timeout
self.request_timeout = request_timeout
self.follow_redirects = follow_redirects
@@ -356,29 +356,32 @@ class HTTPResponse(object):
* code: numeric HTTP status code, e.g. 200 or 404
* reason: human-readable reason phrase describing the status code
(with curl_httpclient, this is a default value rather than the
server's actual response)
(with curl_httpclient, this is a default value rather than the
server's actual response)
* headers: httputil.HTTPHeaders object
* headers: `tornado.httputil.HTTPHeaders` object
* buffer: cStringIO object for response body
* buffer: ``cStringIO`` object for response body
* body: response body as string (created on demand from self.buffer)
* body: response body as string (created on demand from ``self.buffer``)
* error: Exception object, if any
* request_time: seconds from request start to finish
* time_info: dictionary of diagnostic timing information from the request.
Available data are subject to change, but currently uses timings
available from http://curl.haxx.se/libcurl/c/curl_easy_getinfo.html,
plus 'queue', which is the delay (if any) introduced by waiting for
a slot under AsyncHTTPClient's max_clients setting.
Available data are subject to change, but currently uses timings
available from http://curl.haxx.se/libcurl/c/curl_easy_getinfo.html,
plus ``queue``, which is the delay (if any) introduced by waiting for
a slot under `AsyncHTTPClient`'s ``max_clients`` setting.
"""
def __init__(self, request, code, headers=None, buffer=None,
effective_url=None, error=None, request_time=None,
time_info=None, reason=None):
self.request = request
if isinstance(request, _RequestProxy):
self.request = request.request
else:
self.request = request
self.code = code
self.reason = reason or httputil.responses.get(code, "Unknown")
if headers is not None:
@@ -426,13 +429,13 @@ class HTTPError(Exception):
Attributes:
code - HTTP error integer error code, e.g. 404. Error code 599 is
used when no HTTP response was received, e.g. for a timeout.
* ``code`` - HTTP error integer error code, e.g. 404. Error code 599 is
used when no HTTP response was received, e.g. for a timeout.
response - HTTPResponse object, if any.
* ``response`` - `HTTPResponse` object, if any.
Note that if follow_redirects is False, redirects become HTTPErrors,
and you can look at error.response.headers['Location'] to see the
Note that if ``follow_redirects`` is False, redirects become HTTPErrors,
and you can look at ``error.response.headers['Location']`` to see the
destination of the redirect.
"""
def __init__(self, code, message=None, response=None):