dinislage/CouchPotatoServer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update to Tornado 3.0

Browse Source
This commit is contained in:
Ruud
2013-04-19 14:49:27 +02:00
parent 4f6b31d14a
commit 0e90739786
29 changed files with 1932 additions and 1166 deletions
Download Patch File Download Diff File Expand all files Collapse all files
couchpotato/core/_base/_core/main.py
+3 -3
View File
@@ -79,7 +79,7 @@ class Core(Plugin):
def shutdown():
self.initShutdown()
IOLoop.instance().add_callback(shutdown)
IOLoop.current().add_callback(shutdown)
return 'shutdown'
@@ -89,7 +89,7 @@ class Core(Plugin):
def restart():
self.initShutdown(restart = True)
IOLoop.instance().add_callback(restart)
IOLoop.current().add_callback(restart)
return 'restarting'
@@ -128,7 +128,7 @@ class Core(Plugin):
log.debug('Save to shutdown/restart')
try:
IOLoop.instance().stop()
IOLoop.current().stop()
except RuntimeError:
pass
except:
couchpotato/runner.py
+2 -1
View File
@@ -241,7 +241,8 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En
from tornado.ioloop import IOLoop
web_container = WSGIContainer(app)
web_container._log = _log
loop = IOLoop.instance()
loop = IOLoop.current()
application = Application([
(r'%s/api/%s/nonblock/(.*)/' % (url_base, api_key), NonBlockHandler),
libs/tornado/__init__.py
+4 -4
View File
@@ -23,7 +23,7 @@ from __future__ import absolute_import, division, print_function, with_statement
# version_info is a four-tuple for programmatic comparison. The first
# three numbers are the components of the version number. The fourth
# is zero for an official release, positive for a development branch,
# or negative for a release candidate (after the base version number
# has been incremented)
version = "2.4.post3"
version_info = (2, 4, 0, 3)
# or negative for a release candidate or beta (after the base version
# number has been incremented)
version = "3.1.dev2"
version_info = (3, 1, 0, -99)
libs/tornado/auth.py
+263 -209
View File
@@ -14,15 +14,19 @@
# License for the specific language governing permissions and limitations
# under the License.
"""Implementations of various third-party authentication schemes.
"""This module contains implementations of various third-party
authentication schemes.
All the classes in this file are class Mixins designed to be used with
web.py RequestHandler classes. The primary methods for each service are
authenticate_redirect(), authorize_redirect(), and get_authenticated_user().
The former should be called to redirect the user to, e.g., the OpenID
authentication page on the third party service, and the latter should
be called upon return to get the user data from the data returned by
the third party service.
All the classes in this file are class mixins designed to be used with
the `tornado.web.RequestHandler` class. They are used in two ways:
* On a login handler, use methods such as ``authenticate_redirect()``,
``authorize_redirect()``, and ``get_authenticated_user()`` to
establish the user's identity and store authentication tokens to your
database and/or cookies.
* In non-login handlers, use methods such as ``facebook_request()``
or ``twitter_request()`` to use the authentication tokens to make
requests to the respective services.
They all take slightly different arguments due to the fact all these
services implement authentication and authorization slightly differently.
@@ -30,18 +34,16 @@ See the individual service classes below for complete documentation.
Example usage for Google OpenID::
class GoogleHandler(tornado.web.RequestHandler, tornado.auth.GoogleMixin):
class GoogleLoginHandler(tornado.web.RequestHandler,
tornado.auth.GoogleMixin):
@tornado.web.asynchronous
@tornado.gen.coroutine
def get(self):
if self.get_argument("openid.mode", None):
self.get_authenticated_user(self.async_callback(self._on_auth))
return
self.authenticate_redirect()
def _on_auth(self, user):
if not user:
raise tornado.web.HTTPError(500, "Google auth failed")
# Save the user with, e.g., set_secure_cookie()
user = yield self.get_authenticated_user()
# Save the user with e.g. set_secure_cookie()
else:
self.authenticate_redirect()
"""
from __future__ import absolute_import, division, print_function, with_statement
@@ -72,9 +74,11 @@ try:
except ImportError:
import urllib as urllib_parse # py2
class AuthError(Exception):
pass
def _auth_future_to_callback(callback, future):
try:
result = future.result()
@@ -83,6 +87,7 @@ def _auth_future_to_callback(callback, future):
result = None
callback(result)
def _auth_return_future(f):
"""Similar to tornado.concurrent.return_future, but uses the auth
module's legacy callback interface.
@@ -91,6 +96,7 @@ def _auth_return_future(f):
inside the function will actually be a future.
"""
replacer = ArgReplacer(f, 'callback')
@functools.wraps(f)
def wrapper(*args, **kwargs):
future = Future()
@@ -102,17 +108,23 @@ def _auth_return_future(f):
return future
return wrapper
class OpenIdMixin(object):
"""Abstract implementation of OpenID and Attribute Exchange.
See GoogleMixin below for example implementations.
See `GoogleMixin` below for a customized example (which also
includes OAuth support).
Class attributes:
* ``_OPENID_ENDPOINT``: the identity provider's URI.
"""
def authenticate_redirect(self, callback_uri=None,
ax_attrs=["name", "email", "language", "username"]):
"""Returns the authentication URL for this service.
"""Redirects to the authentication URL for this service.
After authentication, the service will redirect back to the given
callback URI.
callback URI with additional parameters including ``openid.mode``.
We request the given attributes for the authenticated user by
default (name, email, language, and username). If you don't need
@@ -128,8 +140,12 @@ class OpenIdMixin(object):
"""Fetches the authenticated user data upon redirect.
This method should be called by the handler that receives the
redirect from the authenticate_redirect() or authorize_redirect()
methods.
redirect from the `authenticate_redirect()` method (which is
often the same as the one that calls it; in that case you would
call `get_authenticated_user` if the ``openid.mode`` parameter
is present and `authenticate_redirect` if it is not).
The result of this method will generally be used to set a cookie.
"""
# Verify the OpenID response via direct request to the OP
args = dict((k, v[-1]) for k, v in self.request.arguments.items())
@@ -192,8 +208,8 @@ class OpenIdMixin(object):
def _on_authentication_verified(self, future, response):
if response.error or b"is_valid:true" not in response.body:
future.set_exception(AuthError(
"Invalid OpenID response: %s" % (response.error or
response.body)))
"Invalid OpenID response: %s" % (response.error or
response.body)))
return
# Make sure we got back at least an email from attribute exchange
@@ -250,32 +266,44 @@ class OpenIdMixin(object):
future.set_result(user)
def get_auth_http_client(self):
"""Returns the AsyncHTTPClient instance to be used for auth requests.
"""Returns the `.AsyncHTTPClient` instance to be used for auth requests.
May be overridden by subclasses to use an http client other than
May be overridden by subclasses to use an HTTP client other than
the default.
"""
return httpclient.AsyncHTTPClient()
class OAuthMixin(object):
"""Abstract implementation of OAuth.
"""Abstract implementation of OAuth 1.0 and 1.0a.
See TwitterMixin and FriendFeedMixin below for example implementations.
See `TwitterMixin` and `FriendFeedMixin` below for example implementations,
or `GoogleMixin` for an OAuth/OpenID hybrid.
Class attributes:
* ``_OAUTH_AUTHORIZE_URL``: The service's OAuth authorization url.
* ``_OAUTH_ACCESS_TOKEN_URL``: The service's OAuth access token url.
* ``_OAUTH_VERSION``: May be either "1.0" or "1.0a".
* ``_OAUTH_NO_CALLBACKS``: Set this to True if the service requires
advance registration of callbacks.
Subclasses must also override the `_oauth_get_user_future` and
`_oauth_consumer_token` methods.
"""
def authorize_redirect(self, callback_uri=None, extra_params=None,
http_client=None):
"""Redirects the user to obtain OAuth authorization for this service.
Twitter and FriendFeed both require that you register a Callback
URL with your application. You should call this method to log the
user in, and then call get_authenticated_user() in the handler
you registered as your Callback URL to complete the authorization
process.
The ``callback_uri`` may be omitted if you have previously
registered a callback URI with the third-party service. For some
sevices (including Twitter and Friendfeed), you must use a
previously-registered callback URI and cannot specify a callback
via this method.
This method sets a cookie called _oauth_request_token which is
subsequently used (and cleared) in get_authenticated_user for
This method sets a cookie called ``_oauth_request_token`` which is
subsequently used (and cleared) in `get_authenticated_user` for
security purposes.
"""
if callback_uri and getattr(self, "_OAUTH_NO_CALLBACKS", False):
@@ -299,15 +327,15 @@ class OAuthMixin(object):
@_auth_return_future
def get_authenticated_user(self, callback, http_client=None):
"""Gets the OAuth authorized user and access token on callback.
This method should be called from the handler for your registered
OAuth Callback URL to complete the registration process. We call
callback with the authenticated user, which in addition to standard
attributes like 'name' includes the 'access_key' attribute, which
contains the OAuth access you can use to make authorized requests
to this service on behalf of the user.
"""Gets the OAuth authorized user and access token.
This method should be called from the handler for your
OAuth callback URL to complete the registration process. We run the
callback with the authenticated user dictionary. This dictionary
will contain an ``access_key`` which can be used to make authorized
requests to this service on behalf of the user. The dictionary will
also contain other fields such as ``name``, depending on the service
used.
"""
future = callback
request_key = escape.utf8(self.get_argument("oauth_token"))
@@ -315,13 +343,13 @@ class OAuthMixin(object):
request_cookie = self.get_cookie("_oauth_request_token")
if not request_cookie:
future.set_exception(AuthError(
"Missing OAuth request token cookie"))
"Missing OAuth request token cookie"))
return
self.clear_cookie("_oauth_request_token")
cookie_key, cookie_secret = [base64.b64decode(escape.utf8(i)) for i in request_cookie.split("|")]
if cookie_key != request_key:
future.set_exception(AuthError(
"Request token does not match cookie"))
"Request token does not match cookie"))
return
token = dict(key=cookie_key, secret=cookie_secret)
if oauth_verifier:
@@ -339,7 +367,7 @@ class OAuthMixin(object):
oauth_signature_method="HMAC-SHA1",
oauth_timestamp=str(int(time.time())),
oauth_nonce=escape.to_basestring(binascii.b2a_hex(uuid.uuid4().bytes)),
oauth_version=getattr(self, "_OAUTH_VERSION", "1.0a"),
oauth_version="1.0",
)
if getattr(self, "_OAUTH_VERSION", "1.0a") == "1.0a":
if callback_uri == "oob":
@@ -360,8 +388,8 @@ class OAuthMixin(object):
if response.error:
raise Exception("Could not get request token")
request_token = _oauth_parse_response(response.body)
data = (base64.b64encode(request_token["key"]) + b"|" +
base64.b64encode(request_token["secret"]))
data = (base64.b64encode(escape.utf8(request_token["key"])) + b"|" +
base64.b64encode(escape.utf8(request_token["secret"])))
self.set_cookie("_oauth_request_token", data)
args = dict(oauth_token=request_token["key"])
if callback_uri == "oob":
@@ -381,7 +409,7 @@ class OAuthMixin(object):
oauth_signature_method="HMAC-SHA1",
oauth_timestamp=str(int(time.time())),
oauth_nonce=escape.to_basestring(binascii.b2a_hex(uuid.uuid4().bytes)),
oauth_version=getattr(self, "_OAUTH_VERSION", "1.0a"),
oauth_version="1.0",
)
if "verifier" in request_token:
args["oauth_verifier"] = request_token["verifier"]
@@ -405,8 +433,29 @@ class OAuthMixin(object):
self._oauth_get_user_future(access_token).add_done_callback(
self.async_callback(self._on_oauth_get_user, access_token, future))
def _oauth_consumer_token(self):
"""Subclasses must override this to return their OAuth consumer keys.
The return value should be a `dict` with keys ``key`` and ``secret``.
"""
raise NotImplementedError()
@return_future
def _oauth_get_user_future(self, access_token, callback):
"""Subclasses must override this to get basic information about the
user.
Should return a `.Future` whose result is a dictionary
containing information about the user, which may have been
retrieved by using ``access_token`` to make a request to the
service.
The access token will be added to the returned dictionary to make
the result of `get_authenticated_user`.
For backwards compatibility, the callback-based ``_oauth_get_user``
method is also supported.
"""
# By default, call the old-style _oauth_get_user, but new code
# should override this method instead.
self._oauth_get_user(access_token, callback)
@@ -439,7 +488,7 @@ class OAuthMixin(object):
oauth_signature_method="HMAC-SHA1",
oauth_timestamp=str(int(time.time())),
oauth_nonce=escape.to_basestring(binascii.b2a_hex(uuid.uuid4().bytes)),
oauth_version=getattr(self, "_OAUTH_VERSION", "1.0a"),
oauth_version="1.0",
)
args = {}
args.update(base_args)
@@ -450,30 +499,38 @@ class OAuthMixin(object):
else:
signature = _oauth_signature(consumer_token, method, url, args,
access_token)
base_args["oauth_signature"] = signature
base_args["oauth_signature"] = escape.to_basestring(signature)
return base_args
def get_auth_http_client(self):
"""Returns the AsyncHTTPClient instance to be used for auth requests.
"""Returns the `.AsyncHTTPClient` instance to be used for auth requests.
May be overridden by subclasses to use an http client other than
May be overridden by subclasses to use an HTTP client other than
the default.
"""
return httpclient.AsyncHTTPClient()
class OAuth2Mixin(object):
"""Abstract implementation of OAuth v 2."""
"""Abstract implementation of OAuth 2.0.
See `FacebookGraphMixin` below for an example implementation.
Class attributes:
* ``_OAUTH_AUTHORIZE_URL``: The service's authorization url.
* ``_OAUTH_ACCESS_TOKEN_URL``: The service's access token url.
"""
def authorize_redirect(self, redirect_uri=None, client_id=None,
client_secret=None, extra_params=None):
"""Redirects the user to obtain OAuth authorization for this service.
Some providers require that you register a Callback
URL with your application. You should call this method to log the
user in, and then call get_authenticated_user() in the handler
you registered as your Callback URL to complete the authorization
process.
Some providers require that you register a redirect URL with
your application instead of passing one via this method. You
should call this method to log the user in, and then call
``get_authenticated_user`` in the handler for your
redirect URL to complete the authorization process.
"""
args = {
"redirect_uri": redirect_uri,
@@ -503,35 +560,30 @@ class TwitterMixin(OAuthMixin):
"""Twitter OAuth authentication.
To authenticate with Twitter, register your application with
Twitter at http://twitter.com/apps. Then copy your Consumer Key and
Consumer Secret to the application settings 'twitter_consumer_key' and
'twitter_consumer_secret'. Use this Mixin on the handler for the URL
you registered as your application's Callback URL.
Twitter at http://twitter.com/apps. Then copy your Consumer Key
and Consumer Secret to the application
`~tornado.web.Application.settings` ``twitter_consumer_key`` and
``twitter_consumer_secret``. Use this mixin on the handler for the
URL you registered as your application's callback URL.
When your application is set up, you can use this Mixin like this
When your application is set up, you can use this mixin like this
to authenticate the user with Twitter and get access to their stream::
class TwitterHandler(tornado.web.RequestHandler,
tornado.auth.TwitterMixin):
class TwitterLoginHandler(tornado.web.RequestHandler,
tornado.auth.TwitterMixin):
@tornado.web.asynchronous
@tornado.gen.coroutine
def get(self):
if self.get_argument("oauth_token", None):
self.get_authenticated_user(self.async_callback(self._on_auth))
return
self.authorize_redirect()
user = yield self.get_authenticated_user()
# Save the user using e.g. set_secure_cookie()
else:
self.authorize_redirect()
def _on_auth(self, user):
if not user:
raise tornado.web.HTTPError(500, "Twitter auth failed")
# Save the user using, e.g., set_secure_cookie()
The user object returned by get_authenticated_user() includes the
attributes 'username', 'name', and all of the custom Twitter user
attributes describe at
http://apiwiki.twitter.com/Twitter-REST-API-Method%3A-users%C2%A0show
in addition to 'access_token'. You should save the access token with
the user; it is required to make requests on behalf of the user later
with twitter_request().
The user object returned by `~OAuthMixin.get_authenticated_user`
includes the attributes ``username``, ``name``, ``access_token``,
and all of the custom Twitter user attributes described at
https://dev.twitter.com/docs/api/1.1/get/users/show
"""
_OAUTH_REQUEST_TOKEN_URL = "http://api.twitter.com/oauth/request_token"
_OAUTH_ACCESS_TOKEN_URL = "http://api.twitter.com/oauth/access_token"
@@ -541,7 +593,8 @@ class TwitterMixin(OAuthMixin):
_TWITTER_BASE_URL = "http://api.twitter.com/1"
def authenticate_redirect(self, callback_uri=None):
"""Just like authorize_redirect(), but auto-redirects if authorized.
"""Just like `~OAuthMixin.authorize_redirect`, but
auto-redirects if authorized.
This is generally the right interface to use if you are using
Twitter for single-sign on.
@@ -553,35 +606,33 @@ class TwitterMixin(OAuthMixin):
@_auth_return_future
def twitter_request(self, path, callback=None, access_token=None,
post_args=None, **args):
"""Fetches the given API path, e.g., "/statuses/user_timeline/btaylor"
"""Fetches the given API path, e.g., ``/statuses/user_timeline/btaylor``
The path should not include the format (we automatically append
".json" and parse the JSON output).
The path should not include the format or API version number.
(we automatically use JSON format and API version 1).
If the request is a POST, post_args should be provided. Query
If the request is a POST, ``post_args`` should be provided. Query
string arguments should be given as keyword arguments.
All the Twitter methods are documented at
http://apiwiki.twitter.com/Twitter-API-Documentation.
All the Twitter methods are documented at http://dev.twitter.com/
Many methods require an OAuth access token which you can obtain
through authorize_redirect() and get_authenticated_user(). The
user returned through that process includes an 'access_token'
attribute that can be used to make authenticated requests via
this method. Example usage::
Many methods require an OAuth access token which you can
obtain through `~OAuthMixin.authorize_redirect` and
`~OAuthMixin.get_authenticated_user`. The user returned through that
process includes an 'access_token' attribute that can be used
to make authenticated requests via this method. Example
usage::
class MainHandler(tornado.web.RequestHandler,
tornado.auth.TwitterMixin):
@tornado.web.authenticated
@tornado.web.asynchronous
@tornado.gen.coroutine
def get(self):
self.twitter_request(
new_entry = yield self.twitter_request(
"/statuses/update",
post_args={"status": "Testing Tornado Web Server"},
access_token=user["access_token"],
callback=self.async_callback(self._on_post))
def _on_post(self, new_entry):
access_token=self.current_user["access_token"])
if not new_entry:
# Call failed; perhaps missing permission?
self.authorize_redirect()
@@ -617,8 +668,8 @@ class TwitterMixin(OAuthMixin):
def _on_twitter_request(self, future, response):
if response.error:
future.set_exception(AuthError(
"Error response %s fetching %s" % (response.error,
response.request.url)))
"Error response %s fetching %s" % (response.error,
response.request.url)))
return
future.set_result(escape.json_decode(response.body))
@@ -629,49 +680,45 @@ class TwitterMixin(OAuthMixin):
key=self.settings["twitter_consumer_key"],
secret=self.settings["twitter_consumer_secret"])
@return_future
@gen.engine
def _oauth_get_user_future(self, access_token, callback):
@gen.coroutine
def _oauth_get_user_future(self, access_token):
user = yield self.twitter_request(
"/users/show/" + escape.native_str(access_token[b"screen_name"]),
"/users/show/" + escape.native_str(access_token["screen_name"]),
access_token=access_token)
if user:
user["username"] = user["screen_name"]
callback(user)
raise gen.Return(user)
class FriendFeedMixin(OAuthMixin):
"""FriendFeed OAuth authentication.
To authenticate with FriendFeed, register your application with
FriendFeed at http://friendfeed.com/api/applications. Then
copy your Consumer Key and Consumer Secret to the application settings
'friendfeed_consumer_key' and 'friendfeed_consumer_secret'. Use
this Mixin on the handler for the URL you registered as your
application's Callback URL.
FriendFeed at http://friendfeed.com/api/applications. Then copy
your Consumer Key and Consumer Secret to the application
`~tornado.web.Application.settings` ``friendfeed_consumer_key``
and ``friendfeed_consumer_secret``. Use this mixin on the handler
for the URL you registered as your application's Callback URL.
When your application is set up, you can use this Mixin like this
When your application is set up, you can use this mixin like this
to authenticate the user with FriendFeed and get access to their feed::
class FriendFeedHandler(tornado.web.RequestHandler,
tornado.auth.FriendFeedMixin):
class FriendFeedLoginHandler(tornado.web.RequestHandler,
tornado.auth.FriendFeedMixin):
@tornado.web.asynchronous
@tornado.gen.coroutine
def get(self):
if self.get_argument("oauth_token", None):
self.get_authenticated_user(self.async_callback(self._on_auth))
return
self.authorize_redirect()
user = yield self.get_authenticated_user()
# Save the user using e.g. set_secure_cookie()
else:
self.authorize_redirect()
def _on_auth(self, user):
if not user:
raise tornado.web.HTTPError(500, "FriendFeed auth failed")
# Save the user using, e.g., set_secure_cookie()
The user object returned by get_authenticated_user() includes the
attributes 'username', 'name', and 'description' in addition to
'access_token'. You should save the access token with the user;
The user object returned by `~OAuthMixin.get_authenticated_user()` includes the
attributes ``username``, ``name``, and ``description`` in addition to
``access_token``. You should save the access token with the user;
it is required to make requests on behalf of the user later with
friendfeed_request().
`friendfeed_request()`.
"""
_OAUTH_VERSION = "1.0"
_OAUTH_REQUEST_TOKEN_URL = "https://friendfeed.com/account/oauth/request_token"
@@ -685,30 +732,32 @@ class FriendFeedMixin(OAuthMixin):
post_args=None, **args):
"""Fetches the given relative API path, e.g., "/bret/friends"
If the request is a POST, post_args should be provided. Query
If the request is a POST, ``post_args`` should be provided. Query
string arguments should be given as keyword arguments.
All the FriendFeed methods are documented at
http://friendfeed.com/api/documentation.
Many methods require an OAuth access token which you can obtain
through authorize_redirect() and get_authenticated_user(). The
user returned through that process includes an 'access_token'
attribute that can be used to make authenticated requests via
this method. Example usage::
Many methods require an OAuth access token which you can
obtain through `~OAuthMixin.authorize_redirect` and
`~OAuthMixin.get_authenticated_user`. The user returned
through that process includes an ``access_token`` attribute that
can be used to make authenticated requests via this
method.
Example usage::
class MainHandler(tornado.web.RequestHandler,
tornado.auth.FriendFeedMixin):
@tornado.web.authenticated
@tornado.web.asynchronous
@tornado.gen.coroutine
def get(self):
self.friendfeed_request(
new_entry = yield self.friendfeed_request(
"/entry",
post_args={"body": "Testing Tornado Web Server"},
access_token=self.current_user["access_token"],
callback=self.async_callback(self._on_post))
access_token=self.current_user["access_token"])
def _on_post(self, new_entry):
if not new_entry:
# Call failed; perhaps missing permission?
self.authorize_redirect()
@@ -739,8 +788,8 @@ class FriendFeedMixin(OAuthMixin):
def _on_friendfeed_request(self, future, response):
if response.error:
future.set_exception(AuthError(
"Error response %s fetching %s" % (response.error,
response.request.url)))
"Error response %s fetching %s" % (response.error,
response.request.url)))
return
future.set_result(escape.json_decode(response.body))
@@ -751,9 +800,8 @@ class FriendFeedMixin(OAuthMixin):
key=self.settings["friendfeed_consumer_key"],
secret=self.settings["friendfeed_consumer_secret"])
@return_future
@gen.engine
def _oauth_get_user(self, access_token, callback):
@gen.coroutine
def _oauth_get_user_future(self, access_token, callback):
user = yield self.friendfeed_request(
"/feedinfo/" + access_token["username"],
include="id,name,description", access_token=access_token)
@@ -770,26 +818,30 @@ class FriendFeedMixin(OAuthMixin):
class GoogleMixin(OpenIdMixin, OAuthMixin):
"""Google Open ID / OAuth authentication.
No application registration is necessary to use Google for authentication
or to access Google resources on behalf of a user. To authenticate with
Google, redirect with authenticate_redirect(). On return, parse the
response with get_authenticated_user(). We send a dict containing the
values for the user, including 'email', 'name', and 'locale'.
No application registration is necessary to use Google for
authentication or to access Google resources on behalf of a user.
Google implements both OpenID and OAuth in a hybrid mode. If you
just need the user's identity, use
`~OpenIdMixin.authenticate_redirect`. If you need to make
requests to Google on behalf of the user, use
`authorize_redirect`. On return, parse the response with
`~OpenIdMixin.get_authenticated_user`. We send a dict containing
the values for the user, including ``email``, ``name``, and
``locale``.
Example usage::
class GoogleHandler(tornado.web.RequestHandler, tornado.auth.GoogleMixin):
class GoogleLoginHandler(tornado.web.RequestHandler,
tornado.auth.GoogleMixin):
@tornado.web.asynchronous
@tornado.gen.coroutine
def get(self):
if self.get_argument("openid.mode", None):
self.get_authenticated_user(self.async_callback(self._on_auth))
return
self.authenticate_redirect()
def _on_auth(self, user):
if not user:
raise tornado.web.HTTPError(500, "Google auth failed")
# Save the user with, e.g., set_secure_cookie()
user = yield self.get_authenticated_user()
# Save the user with e.g. set_secure_cookie()
else:
self.authenticate_redirect()
"""
_OPENID_ENDPOINT = "https://www.google.com/accounts/o8/ud"
_OAUTH_ACCESS_TOKEN_URL = "https://www.google.com/accounts/OAuthGetAccessToken"
@@ -798,7 +850,8 @@ class GoogleMixin(OpenIdMixin, OAuthMixin):
ax_attrs=["name", "email", "language", "username"]):
"""Authenticates and authorizes for the given Google resource.
Some of the available resources are:
Some of the available resources which can be used in the ``oauth_scope``
argument are:
* Gmail Contacts - http://www.google.com/m8/feeds/
* Calendar - http://www.google.com/calendar/feeds/
@@ -839,7 +892,7 @@ class GoogleMixin(OpenIdMixin, OAuthMixin):
key=self.settings["google_consumer_key"],
secret=self.settings["google_consumer_secret"])
def _oauth_get_user_future(self, access_token, callback):
def _oauth_get_user_future(self, access_token):
return OpenIdMixin.get_authenticated_user(self)
@@ -853,9 +906,9 @@ class FacebookMixin(object):
To authenticate with Facebook, register your application with
Facebook at http://www.facebook.com/developers/apps.php. Then
copy your API Key and Application Secret to the application settings
'facebook_api_key' and 'facebook_secret'.
``facebook_api_key`` and ``facebook_secret``.
When your application is set up, you can use this Mixin like this
When your application is set up, you can use this mixin like this
to authenticate the user with Facebook::
class FacebookHandler(tornado.web.RequestHandler,
@@ -872,11 +925,11 @@ class FacebookMixin(object):
raise tornado.web.HTTPError(500, "Facebook auth failed")
# Save the user using, e.g., set_secure_cookie()
The user object returned by get_authenticated_user() includes the
attributes 'facebook_uid' and 'name' in addition to session attributes
like 'session_key'. You should save the session key with the user; it is
The user object returned by `get_authenticated_user` includes the
attributes ``facebook_uid`` and ``name`` in addition to session attributes
like ``session_key``. You should save the session key with the user; it is
required to make requests on behalf of the user later with
facebook_request().
`facebook_request`.
"""
def authenticate_redirect(self, callback_uri=None, cancel_uri=None,
extended_permissions=None):
@@ -1029,9 +1082,9 @@ class FacebookMixin(object):
return hashlib.md5(body).hexdigest()
def get_auth_http_client(self):
"""Returns the AsyncHTTPClient instance to be used for auth requests.
"""Returns the `.AsyncHTTPClient` instance to be used for auth requests.
May be overridden by subclasses to use an http client other than
May be overridden by subclasses to use an HTTP client other than
the default.
"""
return httpclient.AsyncHTTPClient()
@@ -1043,6 +1096,7 @@ class FacebookGraphMixin(OAuth2Mixin):
_OAUTH_AUTHORIZE_URL = "https://graph.facebook.com/oauth/authorize?"
_OAUTH_NO_CALLBACKS = False
@_auth_return_future
def get_authenticated_user(self, redirect_uri, client_id, client_secret,
code, callback, extra_fields=None):
"""Handles the login for the Facebook user, returning a user object.
@@ -1051,24 +1105,20 @@ class FacebookGraphMixin(OAuth2Mixin):
class FacebookGraphLoginHandler(LoginHandler, tornado.auth.FacebookGraphMixin):
@tornado.web.asynchronous
@tornado.gen.coroutine
def get(self):
if self.get_argument("code", False):
self.get_authenticated_user(
redirect_uri='/auth/facebookgraph/',
client_id=self.settings["facebook_api_key"],
client_secret=self.settings["facebook_secret"],
code=self.get_argument("code"),
callback=self.async_callback(
self._on_login))
return
self.authorize_redirect(redirect_uri='/auth/facebookgraph/',
client_id=self.settings["facebook_api_key"],
extra_params={"scope": "read_stream,offline_access"})
def _on_login(self, user):
logging.error(user)
self.finish()
user = yield self.get_authenticated_user(
redirect_uri='/auth/facebookgraph/',
client_id=self.settings["facebook_api_key"],
client_secret=self.settings["facebook_secret"],
code=self.get_argument("code"))
# Save the user with e.g. set_secure_cookie
else:
self.authorize_redirect(
redirect_uri='/auth/facebookgraph/',
client_id=self.settings["facebook_api_key"],
extra_params={"scope": "read_stream,offline_access"})
"""
http = self.get_auth_http_client()
args = {
@@ -1088,10 +1138,9 @@ class FacebookGraphMixin(OAuth2Mixin):
client_secret, callback, fields))
def _on_access_token(self, redirect_uri, client_id, client_secret,
callback, fields, response):
future, fields, response):
if response.error:
gen_log.warning('Facebook auth error: %s' % str(response))
callback(None)
future.set_exception(AuthError('Facebook auth error: %s' % str(response)))
return
args = escape.parse_qs_bytes(escape.native_str(response.body))
@@ -1103,14 +1152,14 @@ class FacebookGraphMixin(OAuth2Mixin):
self.facebook_request(
path="/me",
callback=self.async_callback(
self._on_get_user_info, callback, session, fields),
self._on_get_user_info, future, session, fields),
access_token=session["access_token"],
fields=",".join(fields)
)
def _on_get_user_info(self, callback, session, fields, user):
def _on_get_user_info(self, future, session, fields, user):
if user is None:
callback(None)
future.set_result(None)
return
fieldmap = {}
@@ -1118,42 +1167,43 @@ class FacebookGraphMixin(OAuth2Mixin):
fieldmap[field] = user.get(field)
fieldmap.update({"access_token": session["access_token"], "session_expires": session.get("expires")})
callback(fieldmap)
future.set_result(fieldmap)
@_auth_return_future
def facebook_request(self, path, callback, access_token=None,
post_args=None, **args):
"""Fetches the given relative API path, e.g., "/btaylor/picture"
If the request is a POST, post_args should be provided. Query
If the request is a POST, ``post_args`` should be provided. Query
string arguments should be given as keyword arguments.
An introduction to the Facebook Graph API can be found at
http://developers.facebook.com/docs/api
Many methods require an OAuth access token which you can obtain
through authorize_redirect() and get_authenticated_user(). The
user returned through that process includes an 'access_token'
attribute that can be used to make authenticated requests via
this method. Example usage::
Many methods require an OAuth access token which you can
obtain through `~OAuth2Mixin.authorize_redirect` and
`get_authenticated_user`. The user returned through that
process includes an ``access_token`` attribute that can be
used to make authenticated requests via this method.
Example usage::
class MainHandler(tornado.web.RequestHandler,
tornado.auth.FacebookGraphMixin):
@tornado.web.authenticated
@tornado.web.asynchronous
@tornado.gen.coroutine
def get(self):
self.facebook_request(
new_entry = yield self.facebook_request(
"/me/feed",
post_args={"message": "I am posting from my Tornado application!"},
access_token=self.current_user["access_token"],
callback=self.async_callback(self._on_post))
access_token=self.current_user["access_token"])
def _on_post(self, new_entry):
if not new_entry:
# Call failed; perhaps missing permission?
self.authorize_redirect()
return
self.finish("Posted a message!")
"""
url = "https://graph.facebook.com" + path
all_args = {}
@@ -1171,18 +1221,18 @@ class FacebookGraphMixin(OAuth2Mixin):
else:
http.fetch(url, callback=callback)
def _on_facebook_request(self, callback, response):
def _on_facebook_request(self, future, response):
if response.error:
gen_log.warning("Error response %s fetching %s", response.error,
response.request.url)
callback(None)
future.set_exception(AuthError("Error response %s fetching %s",
response.error, response.request.url))
return
callback(escape.json_decode(response.body))
future.set_result(escape.json_decode(response.body))
def get_auth_http_client(self):
"""Returns the AsyncHTTPClient instance to be used for auth requests.
"""Returns the `.AsyncHTTPClient` instance to be used for auth requests.
May be overridden by subclasses to use an http client other than
May be overridden by subclasses to use an HTTP client other than
the default.
"""
return httpclient.AsyncHTTPClient()
@@ -1243,10 +1293,14 @@ def _oauth_escape(val):
def _oauth_parse_response(body):
p = escape.parse_qs(body, keep_blank_values=False)
token = dict(key=p[b"oauth_token"][0], secret=p[b"oauth_token_secret"][0])
# I can't find an officially-defined encoding for oauth responses and
# have never seen anyone use non-ascii. Leave the response in a byte
# string for python 2, and use utf8 on python 3.
body = escape.native_str(body)
p = urlparse.parse_qs(body, keep_blank_values=False)
token = dict(key=p["oauth_token"][0], secret=p["oauth_token_secret"][0])
# Add the extra parameters the Provider included to the token
special = (b"oauth_token", b"oauth_token_secret")
special = ("oauth_token", "oauth_token_secret")
token.update((k, p[k][0]) for k in p if k not in special)
return token
libs/tornado/autoreload.py
+18 -13
View File
@@ -14,15 +14,24 @@
# License for the specific language governing permissions and limitations
# under the License.
"""A module to automatically restart the server when a module is modified.
"""xAutomatically restart the server when a source file is modified.
Most applications should not call this module directly. Instead, pass the
Most applications should not access this module directly. Instead, pass the
keyword argument ``debug=True`` to the `tornado.web.Application` constructor.
This will enable autoreload mode as well as checking for changes to templates
and static resources.
and static resources. Note that restarting is a destructive operation
and any requests in progress will be aborted when the process restarts.
This module depends on IOLoop, so it will not work in WSGI applications
and Google AppEngine. It also will not work correctly when HTTPServer's
This module can also be used as a command-line wrapper around scripts
such as unit test runners. See the `main` method for details.
The command-line wrapper and Application debug modes can be used together.
This combination is encouraged as the wrapper catches syntax errors and
other import-time failures, while debug mode catches changes once
the server has started.
This module depends on `.IOLoop`, so it will not work in WSGI applications
and Google App Engine. It also will not work correctly when `.HTTPServer`'s
multi-process mode is used.
Reloading loses any Python interpreter command-line arguments (e.g. ``-u``)
@@ -94,12 +103,8 @@ _io_loops = weakref.WeakKeyDictionary()
def start(io_loop=None, check_time=500):
"""Restarts the process automatically when a module is modified.
We run on the I/O loop, and restarting is a destructive operation,
so will terminate any pending requests.
"""
io_loop = io_loop or ioloop.IOLoop.instance()
"""Begins watching source files for changes using the given `.IOLoop`. """
io_loop = io_loop or ioloop.IOLoop.current()
if io_loop in _io_loops:
return
_io_loops[io_loop] = True
@@ -137,8 +142,8 @@ def add_reload_hook(fn):
Note that for open file and socket handles it is generally
preferable to set the ``FD_CLOEXEC`` flag (using `fcntl` or
`tornado.platform.auto.set_close_exec`) instead of using a reload
hook to close them.
``tornado.platform.auto.set_close_exec``) instead
of using a reload hook to close them.
"""
_reload_hooks.append(fn)
libs/tornado/concurrent.py
+94 -21
View File
@@ -13,12 +13,21 @@
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Utilities for working with threads and ``Futures``.
``Futures`` are a pattern for concurrent programming introduced in
Python 3.2 in the `concurrent.futures` package (this package has also
been backported to older versions of Python and can be installed with
``pip install futures``). Tornado will use `concurrent.futures.Future` if
it is available; otherwise it will use a compatible class defined in this
module.
"""
from __future__ import absolute_import, division, print_function, with_statement
import functools
import sys
from tornado.stack_context import ExceptionStackContext
from tornado.stack_context import ExceptionStackContext, wrap
from tornado.util import raise_exc_info, ArgReplacer
try:
@@ -26,10 +35,12 @@ try:
except ImportError:
futures = None
class ReturnValueIgnoredError(Exception):
pass
class DummyFuture(object):
class _DummyFuture(object):
def __init__(self):
self._done = False
self._result = None
@@ -87,50 +98,92 @@ class DummyFuture(object):
self._callbacks = None
if futures is None:
Future = DummyFuture
Future = _DummyFuture
else:
Future = futures.Future
class TracebackFuture(Future):
"""Subclass of `Future` which can store a traceback with
exceptions.
The traceback is automatically available in Python 3, but in the
Python 2 futures backport this information is discarded.
"""
def __init__(self):
super(TracebackFuture, self).__init__()
self.__exc_info = None
def exc_info(self):
return self.__exc_info
def set_exc_info(self, exc_info):
"""Traceback-aware replacement for
`~concurrent.futures.Future.set_exception`.
"""
self.__exc_info = exc_info
self.set_exception(exc_info[1])
def result(self):
if self.__exc_info is not None:
raise_exc_info(self.__exc_info)
else:
return super(TracebackFuture, self).result()
class DummyExecutor(object):
def submit(self, fn, *args, **kwargs):
future = Future()
future = TracebackFuture()
try:
future.set_result(fn(*args, **kwargs))
except Exception as e:
future.set_exception(e)
except Exception:
future.set_exc_info(sys.exc_info())
return future
dummy_executor = DummyExecutor()
def run_on_executor(fn):
"""Decorator to run a synchronous method asynchronously on an executor.
The decorated method may be called with a ``callback`` keyword
argument and returns a future.
"""
@functools.wraps(fn)
def wrapper(self, *args, **kwargs):
callback = kwargs.pop("callback", None)
future = self.executor.submit(fn, self, *args, **kwargs)
if callback:
self.io_loop.add_future(future, callback)
self.io_loop.add_future(future,
lambda future: callback(future.result()))
return future
return wrapper
_NO_RESULT = object()
def return_future(f):
"""Decorator to make a function that returns via callback return a `Future`.
"""Decorator to make a function that returns via callback return a
`Future`.
The wrapped function should take a ``callback`` keyword argument
and invoke it with one argument when it has finished. To signal failure,
the function can simply raise an exception (which will be
captured by the `stack_context` and passed along to the `Future`).
captured by the `.StackContext` and passed along to the ``Future``).
From the caller's perspective, the callback argument is optional.
If one is given, it will be invoked when the function is complete
with the `Future` as an argument. If no callback is given, the caller
should use the `Future` to wait for the function to complete
(perhaps by yielding it in a `gen.engine` function, or passing it
to `IOLoop.add_future`).
with `Future.result()` as an argument. If the function fails, the
callback will not be run and an exception will be raised into the
surrounding `.StackContext`.
If no callback is given, the caller should use the ``Future`` to
wait for the function to complete (perhaps by yielding it in a
`.gen.engine` function, or passing it to `.IOLoop.add_future`).
Usage::
@return_future
def future_func(arg1, arg2, callback):
# Do stuff (possibly asynchronous)
@@ -142,19 +195,20 @@ def return_future(f):
callback()
Note that ``@return_future`` and ``@gen.engine`` can be applied to the
same function, provided ``@return_future`` appears first.
same function, provided ``@return_future`` appears first. However,
consider using ``@gen.coroutine`` instead of this combination.
"""
replacer = ArgReplacer(f, 'callback')
@functools.wraps(f)
def wrapper(*args, **kwargs):
future = Future()
callback, args, kwargs = replacer.replace(future.set_result,
args, kwargs)
if callback is not None:
future.add_done_callback(callback)
future = TracebackFuture()
callback, args, kwargs = replacer.replace(
lambda value=_NO_RESULT: future.set_result(value),
args, kwargs)
def handle_error(typ, value, tb):
future.set_exception(value)
future.set_exc_info((typ, value, tb))
return True
exc_info = None
with ExceptionStackContext(handle_error):
@@ -172,9 +226,25 @@ def return_future(f):
# go ahead and raise it to the caller directly without waiting
# for them to inspect the Future.
raise_exc_info(exc_info)
# If the caller passed in a callback, schedule it to be called
# when the future resolves. It is important that this happens
# just before we return the future, or else we risk confusing
# stack contexts with multiple exceptions (one here with the
# immediate exception, and again when the future resolves and
# the callback triggers its exception by calling future.result()).
if callback is not None:
def run_callback(future):
result = future.result()
if result is _NO_RESULT:
callback()
else:
callback(future.result())
future.add_done_callback(wrap(run_callback))
return future
return wrapper
def chain_future(a, b):
"""Chain two futures together so that when one completes, so does the other.
@@ -182,7 +252,10 @@ def chain_future(a, b):
"""
def copy(future):
assert future is a
if a.exception() is not None:
if (isinstance(a, TracebackFuture) and isinstance(b, TracebackFuture)
and a.exc_info() is not None):
b.set_exc_info(a.exc_info())
elif a.exception() is not None:
b.set_exception(a.exception())
else:
b.set_result(a.result())
libs/tornado/curl_httpclient.py
+13 -5
View File
@@ -14,7 +14,7 @@
# License for the specific language governing permissions and limitations
# under the License.
"""Blocking and non-blocking HTTP client implementations using pycurl."""
"""Non-blocking HTTP client implementation using pycurl."""
from __future__ import absolute_import, division, print_function, with_statement
@@ -30,7 +30,8 @@ from tornado.log import gen_log
from tornado import stack_context
from tornado.escape import utf8, native_str
from tornado.httpclient import HTTPRequest, HTTPResponse, HTTPError, AsyncHTTPClient, main, _RequestProxy
from tornado.httpclient import HTTPResponse, HTTPError, AsyncHTTPClient, main
from tornado.util import bytes_type
try:
from io import BytesIO # py3
@@ -171,7 +172,7 @@ class CurlAsyncHTTPClient(AsyncHTTPClient):
# libcurl is ready. After each timeout, resync the scheduled
# timeout with libcurl's current state.
new_timeout = self._multi.timeout()
if new_timeout != -1:
if new_timeout >= 0:
self._set_timeout(new_timeout)
def _handle_force_timeout(self):
@@ -319,7 +320,7 @@ def _curl_setup_request(curl, request, buffer, headers):
write_function = request.streaming_callback
else:
write_function = buffer.write
if type(b'') is type(''): # py2
if bytes_type is str: # py2
curl.setopt(pycurl.WRITEFUNCTION, write_function)
else: # py3
# Upstream pycurl doesn't support py3, but ubuntu 12.10 includes
@@ -410,7 +411,14 @@ def _curl_setup_request(curl, request, buffer, headers):
if request.auth_username is not None:
userpwd = "%s:%s" % (request.auth_username, request.auth_password or '')
curl.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_BASIC)
if request.auth_mode is None or request.auth_mode == "basic":
curl.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_BASIC)
elif request.auth_mode == "digest":
curl.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_DIGEST)
else:
raise ValueError("Unsupported auth_mode %s" % request.auth_mode)
curl.setopt(pycurl.USERPWD, native_str(userpwd))
gen_log.debug("%s %s (username: %r)", request.method, request.url,
request.auth_username)
libs/tornado/escape.py
+16 -15
View File
@@ -28,9 +28,9 @@ import sys
from tornado.util import bytes_type, unicode_type, basestring_type, u
try:
from urllib.parse import parse_qs # py3
from urllib.parse import parse_qs as _parse_qs # py3
except ImportError:
from urlparse import parse_qs # Python 2.6+
from urlparse import parse_qs as _parse_qs # Python 2.6+
try:
import htmlentitydefs # py2
@@ -54,7 +54,7 @@ _XHTML_ESCAPE_DICT = {'&': '&amp;', '<': '&lt;', '>': '&gt;', '"': '&quot;'}
def xhtml_escape(value):
"""Escapes a string so it is valid within XML or XHTML."""
"""Escapes a string so it is valid within HTML or XML."""
return _XHTML_ESCAPE_RE.sub(lambda match: _XHTML_ESCAPE_DICT[match.group(0)],
to_basestring(value))
@@ -72,7 +72,7 @@ def json_encode(value):
# the javscript. Some json libraries do this escaping by default,
# although python's standard library does not, so we do it here.
# http://stackoverflow.com/questions/1580647/json-why-are-forward-slashes-escaped
return json.dumps(recursive_unicode(value)).replace("</", "<\\/")
return json.dumps(value).replace("</", "<\\/")
def json_decode(value):
@@ -86,7 +86,7 @@ def squeeze(value):
def url_escape(value):
"""Returns a valid URL-encoded version of the given value."""
"""Returns a URL-encoded version of the given value."""
return urllib_parse.quote_plus(utf8(value))
# python 3 changed things around enough that we need two separate
@@ -106,7 +106,7 @@ if sys.version_info[0] < 3:
else:
return unicode_type(urllib_parse.unquote_plus(utf8(value)), encoding)
parse_qs_bytes = parse_qs
parse_qs_bytes = _parse_qs
else:
def url_unescape(value, encoding='utf-8'):
"""Decodes the given value from a URL.
@@ -131,8 +131,8 @@ else:
"""
# This is gross, but python3 doesn't give us another way.
# Latin1 is the universal donor of character encodings.
result = parse_qs(qs, keep_blank_values, strict_parsing,
encoding='latin1', errors='strict')
result = _parse_qs(qs, keep_blank_values, strict_parsing,
encoding='latin1', errors='strict')
encoded = {}
for k, v in result.items():
encoded[k] = [i.encode('latin1') for i in v]
@@ -231,9 +231,9 @@ def linkify(text, shorten=False, extra_params="",
Parameters:
shorten: Long urls will be shortened for display.
* ``shorten``: Long urls will be shortened for display.
extra_params: Extra text to include in the link tag, or a callable
* ``extra_params``: Extra text to include in the link tag, or a callable
taking the link as an argument and returning the extra text
e.g. ``linkify(text, extra_params='rel="nofollow" class="external"')``,
or::
@@ -245,12 +245,13 @@ def linkify(text, shorten=False, extra_params="",
return 'class="external" rel="nofollow"'
linkify(text, extra_params=extra_params_cb)
require_protocol: Only linkify urls which include a protocol. If this is
False, urls such as www.facebook.com will also be linkified.
* ``require_protocol``: Only linkify urls which include a protocol. If
this is False, urls such as www.facebook.com will also be linkified.
permitted_protocols: List (or set) of protocols which should be linkified,
e.g. linkify(text, permitted_protocols=["http", "ftp", "mailto"]).
It is very unsafe to include protocols such as "javascript".
* ``permitted_protocols``: List (or set) of protocols which should be
linkified, e.g. ``linkify(text, permitted_protocols=["http", "ftp",
"mailto"])``. It is very unsafe to include protocols such as
``javascript``.
"""
if extra_params and not callable(extra_params):
extra_params = " " + extra_params.strip()
libs/tornado/gen.py
+167 -37
View File
@@ -20,29 +20,41 @@ could be written with ``gen`` as::
class GenAsyncHandler(RequestHandler):
@asynchronous
@gen.engine
@gen.coroutine
def get(self):
http_client = AsyncHTTPClient()
response = yield gen.Task(http_client.fetch, "http://example.com")
response = yield http_client.fetch("http://example.com")
do_something_with_response(response)
self.render("template.html")
`Task` works with any function that takes a ``callback`` keyword
argument. You can also yield a list of ``Tasks``, which will be
Most asynchronous functions in Tornado return a `.Future`;
yielding this object returns its `~.Future.result`.
For functions that do not return ``Futures``, `Task` works with any
function that takes a ``callback`` keyword argument (most Tornado functions
can be used in either style, although the ``Future`` style is preferred
since it is both shorter and provides better exception handling)::
@gen.coroutine
def get(self):
yield gen.Task(AsyncHTTPClient().fetch, "http://example.com")
You can also yield a list of ``Futures`` and/or ``Tasks``, which will be
started at the same time and run in parallel; a list of results will
be returned when they are all finished::
@gen.coroutine
def get(self):
http_client = AsyncHTTPClient()
response1, response2 = yield [gen.Task(http_client.fetch, url1),
gen.Task(http_client.fetch, url2)]
response1, response2 = yield [http_client.fetch(url1),
http_client.fetch(url2)]
For more complicated interfaces, `Task` can be split into two parts:
`Callback` and `Wait`::
class GenAsyncHandler2(RequestHandler):
@asynchronous
@gen.engine
@gen.coroutine
def get(self):
http_client = AsyncHTTPClient()
http_client.fetch("http://example.com",
@@ -70,9 +82,9 @@ import itertools
import sys
import types
from tornado.concurrent import Future
from tornado.concurrent import Future, TracebackFuture
from tornado.ioloop import IOLoop
from tornado.stack_context import ExceptionStackContext
from tornado.stack_context import ExceptionStackContext, wrap
class KeyReuseError(Exception):
@@ -91,18 +103,26 @@ class BadYieldError(Exception):
pass
def engine(func):
"""Decorator for asynchronous generators.
class ReturnValueIgnoredError(Exception):
pass
Any generator that yields objects from this module must be wrapped
in this decorator. The decorator only works on functions that are
already asynchronous. For `~tornado.web.RequestHandler`
``get``/``post``/etc methods, this means that both the
`tornado.web.asynchronous` and `tornado.gen.engine` decorators
must be used (for proper exception handling, ``asynchronous``
should come before ``gen.engine``). In most other cases, it means
that it doesn't make sense to use ``gen.engine`` on functions that
don't already take a callback argument.
def engine(func):
"""Callback-oriented decorator for asynchronous generators.
This is an older interface; for new code that does not need to be
compatible with versions of Tornado older than 3.0 the
`coroutine` decorator is recommended instead.
This decorator is similar to `coroutine`, except it does not
return a `.Future` and the ``callback`` argument is not treated
specially.
In most cases, functions decorated with `engine` should take
a ``callback`` argument and invoke it with their result when
they are finished. One notable exception is the
`~tornado.web.RequestHandler` :ref:`HTTP verb methods <verbs>`,
which use ``self.finish()`` in place of a callback argument.
"""
@functools.wraps(func)
def wrapper(*args, **kwargs):
@@ -116,20 +136,129 @@ def engine(func):
if runner is not None:
return runner.handle_exception(typ, value, tb)
return False
with ExceptionStackContext(handle_exception) as deactivate:
gen = func(*args, **kwargs)
if isinstance(gen, types.GeneratorType):
runner = Runner(gen, deactivate)
runner.run()
return
assert gen is None, gen
deactivate()
with ExceptionStackContext(handle_exception):
try:
result = func(*args, **kwargs)
except (Return, StopIteration) as e:
result = getattr(e, 'value', None)
else:
if isinstance(result, types.GeneratorType):
def final_callback(value):
if value is not None:
raise ReturnValueIgnoredError(
"@gen.engine functions cannot return values: "
"%r" % (value,))
assert value is None
runner = Runner(result, final_callback)
runner.run()
return
if result is not None:
raise ReturnValueIgnoredError(
"@gen.engine functions cannot return values: %r" %
(result,))
# no yield, so we're done
return wrapper
def coroutine(func):
"""Decorator for asynchronous generators.
Any generator that yields objects from this module must be wrapped
in either this decorator or `engine`. These decorators only work
on functions that are already asynchronous. For
`~tornado.web.RequestHandler` :ref:`HTTP verb methods <verbs>` methods, this
means that both the `tornado.web.asynchronous` and
`tornado.gen.coroutine` decorators must be used (for proper
exception handling, ``asynchronous`` should come before
``gen.coroutine``).
Coroutines may "return" by raising the special exception
`Return(value) <Return>`. In Python 3.3+, it is also possible for
the function to simply use the ``return value`` statement (prior to
Python 3.3 generators were not allowed to also return values).
In all versions of Python a coroutine that simply wishes to exit
early may use the ``return`` statement without a value.
Functions with this decorator return a `.Future`. Additionally,
they may be called with a ``callback`` keyword argument, which
will be invoked with the future's result when it resolves. If the
coroutine fails, the callback will not be run and an exception
will be raised into the surrounding `.StackContext`. The
``callback`` argument is not visible inside the decorated
function; it is handled by the decorator itself.
From the caller's perspective, ``@gen.coroutine`` is similar to
the combination of ``@return_future`` and ``@gen.engine``.
"""
@functools.wraps(func)
def wrapper(*args, **kwargs):
runner = None
future = TracebackFuture()
if 'callback' in kwargs:
callback = kwargs.pop('callback')
IOLoop.current().add_future(
future, lambda future: callback(future.result()))
def handle_exception(typ, value, tb):
try:
if runner is not None and runner.handle_exception(typ, value, tb):
return True
except Exception:
typ, value, tb = sys.exc_info()
future.set_exc_info((typ, value, tb))
return True
with ExceptionStackContext(handle_exception):
try:
result = func(*args, **kwargs)
except (Return, StopIteration) as e:
result = getattr(e, 'value', None)
except Exception:
future.set_exc_info(sys.exc_info())
return future
else:
if isinstance(result, types.GeneratorType):
def final_callback(value):
future.set_result(value)
runner = Runner(result, final_callback)
runner.run()
return future
future.set_result(result)
return future
return wrapper
class Return(Exception):
"""Special exception to return a value from a `coroutine`.
If this exception is raised, its value argument is used as the
result of the coroutine::
@gen.coroutine
def fetch_json(url):
response = yield AsyncHTTPClient().fetch(url)
raise gen.Return(json_decode(response.body))
In Python 3.3, this exception is no longer necessary: the ``return``
statement can be used directly to return a value (previously
``yield`` and ``return`` with a value could not be combined in the
same function).
By analogy with the return statement, the value argument is optional,
but it is never necessary to ``raise gen.Return()``. The ``return``
statement can be used with no arguments instead.
"""
def __init__(self, value=None):
super(Return, self).__init__()
self.value = value
class YieldPoint(object):
"""Base class for objects that may be yielded from the generator."""
"""Base class for objects that may be yielded from the generator.
Applications do not normally need to use this class, but it may be
subclassed to provide additional yielding behavior.
"""
def start(self, runner):
"""Called by the runner after the generator has yielded.
@@ -195,7 +324,7 @@ class Wait(YieldPoint):
class WaitAll(YieldPoint):
"""Returns the results of multiple previous `Callbacks`.
"""Returns the results of multiple previous `Callbacks <Callback>`.
The argument is a sequence of `Callback` keys, and the result is
a list of results in the same order.
@@ -291,7 +420,7 @@ class Multi(YieldPoint):
def is_ready(self):
finished = list(itertools.takewhile(
lambda i: i.is_ready(), self.unfinished_children))
lambda i: i.is_ready(), self.unfinished_children))
self.unfinished_children.difference_update(finished)
return not self.unfinished_children
@@ -331,13 +460,13 @@ class Runner(object):
def register_callback(self, key):
"""Adds ``key`` to the list of callbacks."""
if key in self.pending_callbacks:
raise KeyReuseError("key %r is already pending" % key)
raise KeyReuseError("key %r is already pending" % (key,))
self.pending_callbacks.add(key)
def is_ready(self, key):
"""Returns true if a result is available for ``key``."""
if key not in self.pending_callbacks:
raise UnknownKeyError("key %r is not pending" % key)
raise UnknownKeyError("key %r is not pending" % (key,))
return key in self.results
def set_result(self, key, result):
@@ -374,7 +503,7 @@ class Runner(object):
yielded = self.gen.throw(*exc_info)
else:
yielded = self.gen.send(next)
except StopIteration:
except (StopIteration, Return) as e:
self.finished = True
if self.pending_callbacks and not self.had_exception:
# If we ran cleanly without waiting on all callbacks
@@ -384,7 +513,7 @@ class Runner(object):
raise LeakedCallbackError(
"finished without waiting for callbacks %r" %
self.pending_callbacks)
self.final_callback()
self.final_callback(getattr(e, 'value', None))
self.final_callback = None
return
except Exception:
@@ -401,7 +530,8 @@ class Runner(object):
except Exception:
self.exc_info = sys.exc_info()
else:
self.exc_info = (BadYieldError("yielded unknown object %r" % yielded),)
self.exc_info = (BadYieldError(
"yielded unknown object %r" % (yielded,)),)
finally:
self.running = False
@@ -414,7 +544,7 @@ class Runner(object):
else:
result = None
self.set_result(key, result)
return inner
return wrap(inner)
def handle_exception(self, typ, value, tb):
if not self.running and not self.finished:
libs/tornado/httpclient.py
+90 -87
View File
@@ -1,36 +1,35 @@
"""Blocking and non-blocking HTTP client interfaces.
This module defines a common interface shared by two implementations,
`simple_httpclient` and `curl_httpclient`. Applications may either
``simple_httpclient`` and ``curl_httpclient``. Applications may either
instantiate their chosen implementation class directly or use the
`AsyncHTTPClient` class from this module, which selects an implementation
that can be overridden with the `AsyncHTTPClient.configure` method.
The default implementation is `simple_httpclient`, and this is expected
The default implementation is ``simple_httpclient``, and this is expected
to be suitable for most users' needs. However, some applications may wish
to switch to `curl_httpclient` for reasons such as the following:
to switch to ``curl_httpclient`` for reasons such as the following:
* `curl_httpclient` has some features not found in `simple_httpclient`,
* ``curl_httpclient`` has some features not found in ``simple_httpclient``,
including support for HTTP proxies and the ability to use a specified
network interface.
* `curl_httpclient` is more likely to be compatible with sites that are
* ``curl_httpclient`` is more likely to be compatible with sites that are
not-quite-compliant with the HTTP spec, or sites that use little-exercised
features of HTTP.
* `simple_httpclient` only supports SSL on Python 2.6 and above.
* ``curl_httpclient`` is faster.
* `curl_httpclient` is faster
* ``curl_httpclient`` was the default prior to Tornado 2.0.
* `curl_httpclient` was the default prior to Tornado 2.0.
Note that if you are using `curl_httpclient`, it is highly recommended that
Note that if you are using ``curl_httpclient``, it is highly recommended that
you use a recent version of ``libcurl`` and ``pycurl``. Currently the minimum
supported version is 7.18.2, and the recommended version is 7.21.1 or newer.
"""
from __future__ import absolute_import, division, print_function, with_statement
import functools
import time
import weakref
@@ -52,15 +51,15 @@ class HTTPClient(object):
try:
response = http_client.fetch("http://www.google.com/")
print response.body
except httpclient.HTTPError, e:
except httpclient.HTTPError as e:
print "Error:", e
httpclient.close()
"""
def __init__(self, async_client_class=None, **kwargs):
self._io_loop = IOLoop()
if async_client_class is None:
async_client_class = AsyncHTTPClient
self._async_client = async_client_class(self._io_loop, **kwargs)
self._response = None
self._closed = False
def __del__(self):
@@ -82,13 +81,8 @@ class HTTPClient(object):
If an error occurs during the fetch, we raise an `HTTPError`.
"""
def callback(response):
self._response = response
self._io_loop.stop()
self._async_client.fetch(request, callback, **kwargs)
self._io_loop.start()
response = self._response
self._response = None
response = self._io_loop.run_sync(functools.partial(
self._async_client.fetch, request, **kwargs))
response.rethrow()
return response
@@ -98,26 +92,23 @@ class AsyncHTTPClient(Configurable):
Example usage::
import ioloop
def handle_request(response):
if response.error:
print "Error:", response.error
else:
print response.body
ioloop.IOLoop.instance().stop()
http_client = httpclient.AsyncHTTPClient()
http_client = AsyncHTTPClient()
http_client.fetch("http://www.google.com/", handle_request)
ioloop.IOLoop.instance().start()
The constructor for this class is magic in several respects: It actually
creates an instance of an implementation-specific subclass, and instances
are reused as a kind of pseudo-singleton (one per IOLoop). The keyword
argument force_instance=True can be used to suppress this singleton
behavior. Constructor arguments other than io_loop and force_instance
are deprecated. The implementation subclass as well as arguments to
its constructor can be set with the static method configure()
The constructor for this class is magic in several respects: It
actually creates an instance of an implementation-specific
subclass, and instances are reused as a kind of pseudo-singleton
(one per `.IOLoop`). The keyword argument ``force_instance=True``
can be used to suppress this singleton behavior. Constructor
arguments other than ``io_loop`` and ``force_instance`` are
deprecated. The implementation subclass as well as arguments to
its constructor can be set with the static method `configure()`
"""
@classmethod
def configurable_base(cls):
@@ -136,7 +127,7 @@ class AsyncHTTPClient(Configurable):
return getattr(cls, attr_name)
def __new__(cls, io_loop=None, force_instance=False, **kwargs):
io_loop = io_loop or IOLoop.instance()
io_loop = io_loop or IOLoop.current()
if io_loop in cls._async_clients() and not force_instance:
return cls._async_clients()[io_loop]
instance = super(AsyncHTTPClient, cls).__new__(cls, io_loop=io_loop,
@@ -152,25 +143,29 @@ class AsyncHTTPClient(Configurable):
self.defaults.update(defaults)
def close(self):
"""Destroys this http client, freeing any file descriptors used.
"""Destroys this HTTP client, freeing any file descriptors used.
Not needed in normal use, but may be helpful in unittests that
create and destroy http clients. No other methods may be called
on the AsyncHTTPClient after close().
on the `AsyncHTTPClient` after ``close()``.
"""
if self._async_clients().get(self.io_loop) is self:
del self._async_clients()[self.io_loop]
def fetch(self, request, callback=None, **kwargs):
"""Executes a request, calling callback with an `HTTPResponse`.
"""Executes a request, asynchronously returning an `HTTPResponse`.
The request may be either a string URL or an `HTTPRequest` object.
If it is a string, we construct an `HTTPRequest` using any additional
kwargs: ``HTTPRequest(request, **kwargs)``
If an error occurs during the fetch, the HTTPResponse given to the
callback has a non-None error attribute that contains the exception
encountered during the request. You can call response.rethrow() to
throw the exception (if any) in the callback.
This method returns a `.Future` whose result is an
`HTTPResponse`. The ``Future`` wil raise an `HTTPError` if
the request returned a non-200 response code.
If a ``callback`` is given, it will be invoked with the `HTTPResponse`.
In the callback interface, `HTTPError` is not automatically raised.
Instead, you must check the response's ``error`` attribute or
call its `~HTTPResponse.rethrow` method.
"""
if not isinstance(request, HTTPRequest):
request = HTTPRequest(url=request, **kwargs)
@@ -182,6 +177,7 @@ class AsyncHTTPClient(Configurable):
future = Future()
if callback is not None:
callback = stack_context.wrap(callback)
def handle_future(future):
exc = future.exception()
if isinstance(exc, HTTPError) and exc.response is not None:
@@ -194,6 +190,7 @@ class AsyncHTTPClient(Configurable):
response = future.result()
self.io_loop.add_callback(callback, response)
future.add_done_callback(handle_future)
def handle_response(response):
if response.error:
future.set_exception(response.error)
@@ -207,19 +204,19 @@ class AsyncHTTPClient(Configurable):
@classmethod
def configure(cls, impl, **kwargs):
"""Configures the AsyncHTTPClient subclass to use.
"""Configures the `AsyncHTTPClient` subclass to use.
AsyncHTTPClient() actually creates an instance of a subclass.
``AsyncHTTPClient()`` actually creates an instance of a subclass.
This method may be called with either a class object or the
fully-qualified name of such a class (or None to use the default,
SimpleAsyncHTTPClient)
fully-qualified name of such a class (or ``None`` to use the default,
``SimpleAsyncHTTPClient``)
If additional keyword arguments are given, they will be passed
to the constructor of each subclass instance created. The
keyword argument max_clients determines the maximum number of
simultaneous fetch() operations that can execute in parallel
on each IOLoop. Additional arguments may be supported depending
on the implementation class in use.
keyword argument ``max_clients`` determines the maximum number
of simultaneous `~AsyncHTTPClient.fetch()` operations that can
execute in parallel on each `.IOLoop`. Additional arguments
may be supported depending on the implementation class in use.
Example::
@@ -245,7 +242,7 @@ class HTTPRequest(object):
validate_cert=True)
def __init__(self, url, method="GET", headers=None, body=None,
auth_username=None, auth_password=None,
auth_username=None, auth_password=None, auth_mode=None,
connect_timeout=None, request_timeout=None,
if_modified_since=None, follow_redirects=None,
max_redirects=None, user_agent=None, use_gzip=None,
@@ -256,59 +253,61 @@ class HTTPRequest(object):
validate_cert=None, ca_certs=None,
allow_ipv6=None,
client_key=None, client_cert=None):
r"""Creates an `HTTPRequest`.
All parameters except `url` are optional.
r"""All parameters except ``url`` are optional.
:arg string url: URL to fetch
:arg string method: HTTP method, e.g. "GET" or "POST"
:arg headers: Additional HTTP headers to pass on the request
:type headers: `~tornado.httputil.HTTPHeaders` or `dict`
:arg string auth_username: Username for HTTP "Basic" authentication
:arg string auth_password: Password for HTTP "Basic" authentication
:arg string auth_username: Username for HTTP authentication
:arg string auth_password: Password for HTTP authentication
:arg string auth_mode: Authentication mode; default is "basic".
Allowed values are implementation-defined; ``curl_httpclient``
supports "basic" and "digest"; ``simple_httpclient`` only supports
"basic"
:arg float connect_timeout: Timeout for initial connection in seconds
:arg float request_timeout: Timeout for entire request in seconds
:arg datetime if_modified_since: Timestamp for ``If-Modified-Since``
header
:arg if_modified_since: Timestamp for ``If-Modified-Since`` header
:type if_modified_since: `datetime` or `float`
:arg bool follow_redirects: Should redirects be followed automatically
or return the 3xx response?
:arg int max_redirects: Limit for `follow_redirects`
:arg int max_redirects: Limit for ``follow_redirects``
:arg string user_agent: String to send as ``User-Agent`` header
:arg bool use_gzip: Request gzip encoding from the server
:arg string network_interface: Network interface to use for request
:arg callable streaming_callback: If set, `streaming_callback` will
:arg callable streaming_callback: If set, ``streaming_callback`` will
be run with each chunk of data as it is received, and
`~HTTPResponse.body` and `~HTTPResponse.buffer` will be empty in
``HTTPResponse.body`` and ``HTTPResponse.buffer`` will be empty in
the final response.
:arg callable header_callback: If set, `header_callback` will
:arg callable header_callback: If set, ``header_callback`` will
be run with each header line as it is received (including the
first line, e.g. ``HTTP/1.0 200 OK\r\n``, and a final line
containing only ``\r\n``. All lines include the trailing newline
characters). `~HTTPResponse.headers` will be empty in the final
characters). ``HTTPResponse.headers`` will be empty in the final
response. This is most useful in conjunction with
`streaming_callback`, because it's the only way to get access to
``streaming_callback``, because it's the only way to get access to
header data while the request is in progress.
:arg callable prepare_curl_callback: If set, will be called with
a `pycurl.Curl` object to allow the application to make additional
`setopt` calls.
a ``pycurl.Curl`` object to allow the application to make additional
``setopt`` calls.
:arg string proxy_host: HTTP proxy hostname. To use proxies,
`proxy_host` and `proxy_port` must be set; `proxy_username` and
`proxy_pass` are optional. Proxies are currently only support
with `curl_httpclient`.
``proxy_host`` and ``proxy_port`` must be set; ``proxy_username`` and
``proxy_pass`` are optional. Proxies are currently only supported
with ``curl_httpclient``.
:arg int proxy_port: HTTP proxy port
:arg string proxy_username: HTTP proxy username
:arg string proxy_password: HTTP proxy password
:arg bool allow_nonstandard_methods: Allow unknown values for `method`
:arg bool allow_nonstandard_methods: Allow unknown values for ``method``
argument?
:arg bool validate_cert: For HTTPS requests, validate the server's
certificate?
:arg string ca_certs: filename of CA certificates in PEM format,
or None to use defaults. Note that in `curl_httpclient`, if
any request uses a custom `ca_certs` file, they all must (they
don't have to all use the same `ca_certs`, but it's not possible
to mix requests with ca_certs and requests that use the defaults.
or None to use defaults. Note that in ``curl_httpclient``, if
any request uses a custom ``ca_certs`` file, they all must (they
don't have to all use the same ``ca_certs``, but it's not possible
to mix requests with ``ca_certs`` and requests that use the defaults.
:arg bool allow_ipv6: Use IPv6 when available? Default is false in
`simple_httpclient` and true in `curl_httpclient`
``simple_httpclient`` and true in ``curl_httpclient``
:arg string client_key: Filename for client SSL key, if any
:arg string client_cert: Filename for client SSL certificate, if any
"""
@@ -327,6 +326,7 @@ class HTTPRequest(object):
self.body = utf8(body)
self.auth_username = auth_username
self.auth_password = auth_password
self.auth_mode = auth_mode
self.connect_timeout = connect_timeout
self.request_timeout = request_timeout
self.follow_redirects = follow_redirects
@@ -356,29 +356,32 @@ class HTTPResponse(object):
* code: numeric HTTP status code, e.g. 200 or 404
* reason: human-readable reason phrase describing the status code
(with curl_httpclient, this is a default value rather than the
server's actual response)
(with curl_httpclient, this is a default value rather than the
server's actual response)
* headers: httputil.HTTPHeaders object
* headers: `tornado.httputil.HTTPHeaders` object
* buffer: cStringIO object for response body
* buffer: ``cStringIO`` object for response body
* body: response body as string (created on demand from self.buffer)
* body: response body as string (created on demand from ``self.buffer``)
* error: Exception object, if any
* request_time: seconds from request start to finish
* time_info: dictionary of diagnostic timing information from the request.
Available data are subject to change, but currently uses timings
available from http://curl.haxx.se/libcurl/c/curl_easy_getinfo.html,
plus 'queue', which is the delay (if any) introduced by waiting for
a slot under AsyncHTTPClient's max_clients setting.
Available data are subject to change, but currently uses timings
available from http://curl.haxx.se/libcurl/c/curl_easy_getinfo.html,
plus ``queue``, which is the delay (if any) introduced by waiting for
a slot under `AsyncHTTPClient`'s ``max_clients`` setting.
"""
def __init__(self, request, code, headers=None, buffer=None,
effective_url=None, error=None, request_time=None,
time_info=None, reason=None):
self.request = request
if isinstance(request, _RequestProxy):
self.request = request.request
else:
self.request = request
self.code = code
self.reason = reason or httputil.responses.get(code, "Unknown")
if headers is not None:
@@ -426,13 +429,13 @@ class HTTPError(Exception):
Attributes:
code - HTTP error integer error code, e.g. 404. Error code 599 is
used when no HTTP response was received, e.g. for a timeout.
* ``code`` - HTTP error integer error code, e.g. 404. Error code 599 is
used when no HTTP response was received, e.g. for a timeout.
response - HTTPResponse object, if any.
* ``response`` - `HTTPResponse` object, if any.
Note that if follow_redirects is False, redirects become HTTPErrors,
and you can look at error.response.headers['Location'] to see the
Note that if ``follow_redirects`` is False, redirects become HTTPErrors,
and you can look at ``error.response.headers['Location']`` to see the
destination of the redirect.
"""
def __init__(self, code, message=None, response=None):
libs/tornado/httpserver.py
+90 -72
View File
@@ -34,14 +34,15 @@ from tornado.escape import native_str, parse_qs_bytes
from tornado import httputil
from tornado import iostream
from tornado.log import gen_log
from tornado import netutil
from tornado.tcpserver import TCPServer
from tornado import stack_context
from tornado.util import bytes_type
try:
import Cookie # py2
import Cookie # py2
except ImportError:
import http.cookies as Cookie # py3
import http.cookies as Cookie # py3
class HTTPServer(TCPServer):
@@ -54,8 +55,8 @@ class HTTPServer(TCPServer):
requests). A simple example server that echoes back the URI you
requested::
import httpserver
import ioloop
import tornado.httpserver
import tornado.ioloop
def handle_request(request):
message = "You requested %s\n" % request.uri
@@ -63,9 +64,9 @@ class HTTPServer(TCPServer):
len(message), message))
request.finish()
http_server = httpserver.HTTPServer(handle_request)
http_server = tornado.httpserver.HTTPServer(handle_request)
http_server.listen(8888)
ioloop.IOLoop.instance().start()
tornado.ioloop.IOLoop.instance().start()
`HTTPServer` is a very basic connection handler. It parses the request
headers and body, but the request callback is responsible for producing
@@ -92,11 +93,10 @@ class HTTPServer(TCPServer):
if Tornado is run behind an SSL-decoding proxy that does not set one of
the supported ``xheaders``.
`HTTPServer` can serve SSL traffic with Python 2.6+ and OpenSSL.
To make this server serve SSL traffic, send the ssl_options dictionary
To make this server serve SSL traffic, send the ``ssl_options`` dictionary
argument with the arguments required for the `ssl.wrap_socket` method,
including "certfile" and "keyfile". In Python 3.2+ you can pass
an `ssl.SSLContext` object instead of a dict::
including ``certfile`` and ``keyfile``. (In Python 3.2+ you can pass
an `ssl.SSLContext` object instead of a dict)::
HTTPServer(applicaton, ssl_options={
"certfile": os.path.join(data_dir, "mydomain.crt"),
@@ -123,9 +123,9 @@ class HTTPServer(TCPServer):
server.start(0) # Forks multiple sub-processes
IOLoop.instance().start()
When using this interface, an `IOLoop` must *not* be passed
to the `HTTPServer` constructor. `start` will always start
the server on the default singleton `IOLoop`.
When using this interface, an `.IOLoop` must *not* be passed
to the `HTTPServer` constructor. `~.TCPServer.start` will always start
the server on the default singleton `.IOLoop`.
3. `~tornado.tcpserver.TCPServer.add_sockets`: advanced multi-process::
@@ -135,21 +135,21 @@ class HTTPServer(TCPServer):
server.add_sockets(sockets)
IOLoop.instance().start()
The `add_sockets` interface is more complicated, but it can be
used with `tornado.process.fork_processes` to give you more
flexibility in when the fork happens. `add_sockets` can
also be used in single-process servers if you want to create
your listening sockets in some way other than
`tornado.netutil.bind_sockets`.
The `~.TCPServer.add_sockets` interface is more complicated,
but it can be used with `tornado.process.fork_processes` to
give you more flexibility in when the fork happens.
`~.TCPServer.add_sockets` can also be used in single-process
servers if you want to create your listening sockets in some
way other than `tornado.netutil.bind_sockets`.
"""
def __init__(self, request_callback, no_keep_alive=False, io_loop=None,
xheaders=False, ssl_options=None, protocol=None, **kwargs):
def __init__(self, request_callback, no_keep_alive = False, io_loop = None,
xheaders = False, ssl_options = None, protocol = None, **kwargs):
self.request_callback = request_callback
self.no_keep_alive = no_keep_alive
self.xheaders = xheaders
self.protocol = protocol
TCPServer.__init__(self, io_loop=io_loop, ssl_options=ssl_options,
TCPServer.__init__(self, io_loop = io_loop, ssl_options = ssl_options,
**kwargs)
def handle_stream(self, stream, address):
@@ -168,8 +168,8 @@ class HTTPConnection(object):
We parse HTTP headers and bodies, and execute the request callback
until the HTTP conection is closed.
"""
def __init__(self, stream, address, request_callback, no_keep_alive=False,
xheaders=False, protocol=None):
def __init__(self, stream, address, request_callback, no_keep_alive = False,
xheaders = False, protocol = None):
self.stream = stream
self.address = address
# Save the socket's address family now so we know how to
@@ -182,31 +182,51 @@ class HTTPConnection(object):
self.protocol = protocol
self._request = None
self._request_finished = False
self._write_callback = None
self._close_callback = None
# Save stack context here, outside of any request. This keeps
# contexts from one request from leaking into the next.
self._header_callback = stack_context.wrap(self._on_headers)
self.stream.read_until(b"\r\n\r\n", self._header_callback)
def _clear_callbacks(self):
"""Clears the per-request callbacks.
This is run in between requests to allow the previous handler
to be garbage collected (and prevent spurious close callbacks),
and when the connection is closed (to break up cycles and
facilitate garbage collection in cpython).
"""
self._write_callback = None
self._close_callback = None
def set_close_callback(self, callback):
"""Sets a callback that will be run when the connection is closed.
Use this instead of accessing
`HTTPConnection.stream.set_close_callback
<.BaseIOStream.set_close_callback>` directly (which was the
recommended approach prior to Tornado 3.0).
"""
self._close_callback = stack_context.wrap(callback)
self.stream.set_close_callback(self._on_connection_close)
def _on_connection_close(self):
callback = self._close_callback
self._close_callback = None
callback()
if callback: callback()
# Delete any unfinished callbacks to break up reference cycles.
self._write_callback = None
self._header_callback = None
self._clear_callbacks()
def close(self):
self.stream.close()
# Remove this reference to self, which would otherwise cause a
# cycle and delay garbage collection of this connection.
self._header_callback = None
self._clear_callbacks()
def write(self, chunk, callback=None):
def write(self, chunk, callback = None):
"""Writes a chunk of output to the stream."""
assert self._request, "Request closed"
if not self.stream.closed():
@@ -251,6 +271,7 @@ class HTTPConnection(object):
disconnect = True
self._request = None
self._request_finished = False
self._clear_callbacks()
if disconnect:
self.close()
return
@@ -273,7 +294,11 @@ class HTTPConnection(object):
raise _BadRequestException("Malformed HTTP request line")
if not version.startswith("HTTP/"):
raise _BadRequestException("Malformed HTTP version in HTTP Request-Line")
headers = httputil.HTTPHeaders.parse(data[eol:])
try:
headers = httputil.HTTPHeaders.parse(data[eol:])
except ValueError:
# Probably from split() if there was no ':' in the line
raise _BadRequestException("Malformed HTTP headers")
# HTTPRequest wants an IP, not a full socket address
if self.address_family in (socket.AF_INET, socket.AF_INET6):
@@ -283,8 +308,8 @@ class HTTPConnection(object):
remote_ip = '0.0.0.0'
self._request = HTTPRequest(
connection=self, method=method, uri=uri, version=version,
headers=headers, remote_ip=remote_ip, protocol=self.protocol)
connection = self, method = method, uri = uri, version = version,
headers = headers, remote_ip = remote_ip, protocol = self.protocol)
content_length = headers.get("Content-Length")
if content_length:
@@ -339,7 +364,7 @@ class HTTPRequest(object):
.. attribute:: headers
`HTTPHeader` dictionary-like object for request headers. Acts like
`.HTTPHeaders` dictionary-like object for request headers. Acts like
a case-insensitive dictionary with additional methods for repeated
headers.
@@ -349,13 +374,13 @@ class HTTPRequest(object):
.. attribute:: remote_ip
Client's IP address as a string. If `HTTPServer.xheaders` is set,
Client's IP address as a string. If ``HTTPServer.xheaders`` is set,
will pass along the real IP address provided by a load balancer
in the ``X-Real-Ip`` header
.. attribute:: protocol
The protocol used, either "http" or "https". If `HTTPServer.xheaders`
The protocol used, either "http" or "https". If ``HTTPServer.xheaders``
is set, will pass along the protocol used by a load balancer if
reported via an ``X-Scheme`` header.
@@ -369,13 +394,13 @@ class HTTPRequest(object):
maps arguments names to lists of values (to support multiple values
for individual names). Names are of type `str`, while arguments
are byte strings. Note that this is different from
`RequestHandler.get_argument`, which returns argument values as
`.RequestHandler.get_argument`, which returns argument values as
unicode strings.
.. attribute:: files
File uploads are available in the files property, which maps file
names to lists of :class:`HTTPFile`.
names to lists of `.HTTPFile`.
.. attribute:: connection
@@ -384,34 +409,39 @@ class HTTPRequest(object):
are typically kept open in HTTP/1.1, multiple requests can be handled
sequentially on a single connection.
"""
def __init__(self, method, uri, version="HTTP/1.0", headers=None,
body=None, remote_ip=None, protocol=None, host=None,
files=None, connection=None):
def __init__(self, method, uri, version = "HTTP/1.0", headers = None,
body = None, remote_ip = None, protocol = None, host = None,
files = None, connection = None):
self.method = method
self.uri = uri
self.version = version
self.headers = headers or httputil.HTTPHeaders()
self.body = body or ""
# set remote IP and protocol
self.remote_ip = remote_ip
if protocol:
self.protocol = protocol
elif connection and isinstance(connection.stream,
iostream.SSLIOStream):
self.protocol = "https"
else:
self.protocol = "http"
# xheaders can override the defaults
if connection and connection.xheaders:
# Squid uses X-Forwarded-For, others use X-Real-Ip
self.remote_ip = self.headers.get(
"X-Real-Ip", self.headers.get("X-Forwarded-For", remote_ip))
if not self._valid_ip(self.remote_ip):
self.remote_ip = remote_ip
ip = self.headers.get(
"X-Real-Ip", self.headers.get("X-Forwarded-For", self.remote_ip))
if netutil.is_valid_ip(ip):
self.remote_ip = ip
# AWS uses X-Forwarded-Proto
self.protocol = self.headers.get(
"X-Scheme", self.headers.get("X-Forwarded-Proto", protocol))
if self.protocol not in ("http", "https"):
self.protocol = "http"
else:
self.remote_ip = remote_ip
if protocol:
self.protocol = protocol
elif connection and isinstance(connection.stream,
iostream.SSLIOStream):
self.protocol = "https"
else:
self.protocol = "http"
proto = self.headers.get(
"X-Scheme", self.headers.get("X-Forwarded-Proto", self.protocol))
if proto in ("http", "https"):
self.protocol = proto
self.host = host or self.headers.get("Host") or "127.0.0.1"
self.files = files or {}
self.connection = connection
@@ -419,7 +449,7 @@ class HTTPRequest(object):
self._finish_time = None
self.path, sep, self.query = uri.partition('?')
self.arguments = parse_qs_bytes(self.query, keep_blank_values=True)
self.arguments = parse_qs_bytes(self.query, keep_blank_values = True)
def supports_http_1_1(self):
"""Returns True if this request supports HTTP/1.1 semantics"""
@@ -438,10 +468,10 @@ class HTTPRequest(object):
self._cookies = {}
return self._cookies
def write(self, chunk, callback=None):
def write(self, chunk, callback = None):
"""Writes the given chunk to the response stream."""
assert isinstance(chunk, bytes_type)
self.connection.write(chunk, callback=callback)
self.connection.write(chunk, callback = callback)
def finish(self):
"""Finishes this HTTP request on the open connection."""
@@ -459,7 +489,7 @@ class HTTPRequest(object):
else:
return self._finish_time - self._start_time
def get_ssl_certificate(self, binary_form=False):
def get_ssl_certificate(self, binary_form = False):
"""Returns the client's SSL certificate, if any.
To use client certificates, the HTTPServer must have been constructed
@@ -481,7 +511,7 @@ class HTTPRequest(object):
"""
try:
return self.connection.stream.socket.getpeercert(
binary_form=binary_form)
binary_form = binary_form)
except ssl.SSLError:
return None
@@ -491,15 +521,3 @@ class HTTPRequest(object):
args = ", ".join(["%s=%r" % (n, getattr(self, n)) for n in attrs])
return "%s(%s, headers=%s)" % (
self.__class__.__name__, args, dict(self.headers))
def _valid_ip(self, ip):
try:
res = socket.getaddrinfo(ip, 0, socket.AF_UNSPEC,
socket.SOCK_STREAM,
0, socket.AI_NUMERICHOST)
return bool(res)
except socket.gaierror as e:
if e.args[0] == socket.EAI_NONAME:
return False
raise
return True
libs/tornado/httputil.py
+27 -19
View File
@@ -32,6 +32,10 @@ try:
except ImportError:
from http.client import responses # py3
# responses is unused in this file, but we re-export it to other files.
# Reference it so pyflakes doesn't complain.
responses
try:
from urllib import urlencode # py2
except ImportError:
@@ -39,11 +43,12 @@ except ImportError:
class HTTPHeaders(dict):
"""A dictionary that maintains Http-Header-Case for all keys.
"""A dictionary that maintains ``Http-Header-Case`` for all keys.
Supports multiple values per key via a pair of new methods,
add() and get_list(). The regular dictionary interface returns a single
value per key, with multiple values joined by a comma.
`add()` and `get_list()`. The regular dictionary interface
returns a single value per key, with multiple values joined by a
comma.
>>> h = HTTPHeaders({"content-type": "text/html"})
>>> list(h.keys())
@@ -209,13 +214,14 @@ def url_concat(url, args):
class HTTPFile(ObjectDict):
"""Represents an HTTP file. For backwards compatibility, its instance
attributes are also accessible as dictionary keys.
"""Represents a file uploaded via a form.
:ivar filename:
:ivar body:
:ivar content_type: The content_type comes from the provided HTTP header
and should not be trusted outright given that it can be easily forged.
For backwards compatibility, its instance attributes are also
accessible as dictionary keys.
* ``filename``
* ``body``
* ``content_type``
"""
pass
@@ -223,15 +229,15 @@ class HTTPFile(ObjectDict):
def parse_body_arguments(content_type, body, arguments, files):
"""Parses a form request body.
Supports "application/x-www-form-urlencoded" and "multipart/form-data".
The content_type parameter should be a string and body should be
a byte string. The arguments and files parameters are dictionaries
that will be updated with the parsed contents.
Supports ``application/x-www-form-urlencoded`` and
``multipart/form-data``. The ``content_type`` parameter should be
a string and ``body`` should be a byte string. The ``arguments``
and ``files`` parameters are dictionaries that will be updated
with the parsed contents.
"""
if content_type.startswith("application/x-www-form-urlencoded"):
uri_arguments = parse_qs_bytes(native_str(body))
uri_arguments = parse_qs_bytes(native_str(body), keep_blank_values=True)
for name, values in uri_arguments.items():
values = [v for v in values if v]
if values:
arguments.setdefault(name, []).extend(values)
elif content_type.startswith("multipart/form-data"):
@@ -246,9 +252,9 @@ def parse_body_arguments(content_type, body, arguments, files):
def parse_multipart_form_data(boundary, data, arguments, files):
"""Parses a multipart/form-data body.
"""Parses a ``multipart/form-data`` body.
The boundary and data parameters are both byte strings.
The ``boundary`` and ``data`` parameters are both byte strings.
The dictionaries given in the arguments and files parameters
will be updated with the contents of the body.
"""
@@ -294,8 +300,8 @@ def parse_multipart_form_data(boundary, data, arguments, files):
def format_timestamp(ts):
"""Formats a timestamp in the format used by HTTP.
The argument may be a numeric timestamp as returned by `time.time()`,
a time tuple as returned by `time.gmtime()`, or a `datetime.datetime`
The argument may be a numeric timestamp as returned by `time.time`,
a time tuple as returned by `time.gmtime`, or a `datetime.datetime`
object.
>>> format_timestamp(1359312200)
@@ -314,6 +320,8 @@ def format_timestamp(ts):
# _parseparam and _parse_header are copied and modified from python2.7's cgi.py
# The original 2.7 version of this code did not correctly support some
# combinations of semicolons and double quotes.
def _parseparam(s):
while s[:1] == ';':
s = s[1:]
libs/tornado/ioloop.py
+161 -80
View File
@@ -36,11 +36,12 @@ import logging
import numbers
import os
import select
import sys
import threading
import time
import traceback
from tornado.concurrent import DummyFuture
from tornado.concurrent import Future, TracebackFuture
from tornado.log import app_log, gen_log
from tornado import stack_context
from tornado.util import Configurable
@@ -50,11 +51,6 @@ try:
except ImportError:
signal = None
try:
from concurrent import futures
except ImportError:
futures = None
try:
import thread # py2
except ImportError:
@@ -63,14 +59,18 @@ except ImportError:
from tornado.platform.auto import set_close_exec, Waker
class TimeoutError(Exception):
pass
class IOLoop(Configurable):
"""A level-triggered I/O loop.
We use epoll (Linux) or kqueue (BSD and Mac OS X; requires python
2.6+) if they are available, or else we fall back on select(). If
you are implementing a system that needs to handle thousands of
simultaneous connections, you should use a system that supports either
epoll or kqueue.
We use ``epoll`` (Linux) or ``kqueue`` (BSD and Mac OS X) if they
are available, or else we fall back on select(). If you are
implementing a system that needs to handle thousands of
simultaneous connections, you should use a system that supports
either ``epoll`` or ``kqueue``.
Example usage for a simple TCP server::
@@ -125,19 +125,11 @@ class IOLoop(Configurable):
@staticmethod
def instance():
"""Returns a global IOLoop instance.
"""Returns a global `IOLoop` instance.
Most single-threaded applications have a single, global IOLoop.
Use this method instead of passing around IOLoop instances
throughout your code.
A common pattern for classes that depend on IOLoops is to use
a default argument to enable programs with multiple IOLoops
but not require the argument for simpler applications::
class MyClass(object):
def __init__(self, io_loop=None):
self.io_loop = io_loop or IOLoop.instance()
Most applications have a single, global `IOLoop` running on the
main thread. Use this method to get this instance from
another thread. To get the current thread's `IOLoop`, use `current()`.
"""
if not hasattr(IOLoop, "_instance"):
with IOLoop._instance_lock:
@@ -152,27 +144,54 @@ class IOLoop(Configurable):
return hasattr(IOLoop, "_instance")
def install(self):
"""Installs this IOloop object as the singleton instance.
"""Installs this `IOLoop` object as the singleton instance.
This is normally not necessary as `instance()` will create
an IOLoop on demand, but you may want to call `install` to use
a custom subclass of IOLoop.
an `IOLoop` on demand, but you may want to call `install` to use
a custom subclass of `IOLoop`.
"""
assert not IOLoop.initialized()
IOLoop._instance = self
@staticmethod
def current():
"""Returns the current thread's `IOLoop`.
If an `IOLoop` is currently running or has been marked as current
by `make_current`, returns that instance. Otherwise returns
`IOLoop.instance()`, i.e. the main thread's `IOLoop`.
A common pattern for classes that depend on ``IOLoops`` is to use
a default argument to enable programs with multiple ``IOLoops``
but not require the argument for simpler applications::
class MyClass(object):
def __init__(self, io_loop=None):
self.io_loop = io_loop or IOLoop.current()
In general you should use `IOLoop.current` as the default when
constructing an asynchronous object, and use `IOLoop.instance`
when you mean to communicate to the main thread from a different
one.
"""
current = getattr(IOLoop._current, "instance", None)
if current is None:
raise ValueError("no current IOLoop")
return IOLoop.instance()
return current
def make_current(self):
"""Makes this the `IOLoop` for the current thread.
An `IOLoop` automatically becomes current for its thread
when it is started, but it is sometimes useful to call
`make_current` explictly before starting the `IOLoop`,
so that code run at startup time can find the right
instance.
"""
IOLoop._current.instance = self
def clear_current(self):
assert IOLoop._current.instance is self
@staticmethod
def clear_current():
IOLoop._current.instance = None
@classmethod
@@ -195,19 +214,20 @@ class IOLoop(Configurable):
pass
def close(self, all_fds=False):
"""Closes the IOLoop, freeing any resources used.
"""Closes the `IOLoop`, freeing any resources used.
If ``all_fds`` is true, all file descriptors registered on the
IOLoop will be closed (not just the ones created by the IOLoop itself).
IOLoop will be closed (not just the ones created by the
`IOLoop` itself).
Many applications will only use a single IOLoop that runs for the
entire lifetime of the process. In that case closing the IOLoop
Many applications will only use a single `IOLoop` that runs for the
entire lifetime of the process. In that case closing the `IOLoop`
is not necessary since everything will be cleaned up when the
process exits. `IOLoop.close` is provided mainly for scenarios
such as unit tests, which create and destroy a large number of
IOLoops.
``IOLoops``.
An IOLoop must be completely stopped before it can be closed. This
An `IOLoop` must be completely stopped before it can be closed. This
means that `IOLoop.stop()` must be called *and* `IOLoop.start()` must
be allowed to return before attempting to call `IOLoop.close()`.
Therefore the call to `close` will usually appear just after
@@ -216,7 +236,13 @@ class IOLoop(Configurable):
raise NotImplementedError()
def add_handler(self, fd, handler, events):
"""Registers the given handler to receive the given events for fd."""
"""Registers the given handler to receive the given events for fd.
The ``events`` argument is a bitwise or of the constants
``IOLoop.READ``, ``IOLoop.WRITE``, and ``IOLoop.ERROR``.
When an event occurs, ``handler(fd, events)`` will be run.
"""
raise NotImplementedError()
def update_handler(self, fd, events):
@@ -228,28 +254,32 @@ class IOLoop(Configurable):
raise NotImplementedError()
def set_blocking_signal_threshold(self, seconds, action):
"""Sends a signal if the ioloop is blocked for more than s seconds.
"""Sends a signal if the `IOLoop` is blocked for more than
``s`` seconds.
Pass seconds=None to disable. Requires python 2.6 on a unixy
Pass ``seconds=None`` to disable. Requires Python 2.6 on a unixy
platform.
The action parameter is a python signal handler. Read the
documentation for the python 'signal' module for more information.
If action is None, the process will be killed if it is blocked for
too long.
The action parameter is a Python signal handler. Read the
documentation for the `signal` module for more information.
If ``action`` is None, the process will be killed if it is
blocked for too long.
"""
raise NotImplementedError()
def set_blocking_log_threshold(self, seconds):
"""Logs a stack trace if the ioloop is blocked for more than s seconds.
Equivalent to set_blocking_signal_threshold(seconds, self.log_stack)
"""Logs a stack trace if the `IOLoop` is blocked for more than
``s`` seconds.
Equivalent to ``set_blocking_signal_threshold(seconds,
self.log_stack)``
"""
self.set_blocking_signal_threshold(seconds, self.log_stack)
def log_stack(self, signal, frame):
"""Signal handler to log the stack trace of the current thread.
For use with set_blocking_signal_threshold.
For use with `set_blocking_signal_threshold`.
"""
gen_log.warning('IOLoop blocked for %f seconds in\n%s',
self._blocking_signal_threshold,
@@ -258,7 +288,7 @@ class IOLoop(Configurable):
def start(self):
"""Starts the I/O loop.
The loop will run until one of the I/O handlers calls stop(), which
The loop will run until one of the callbacks calls `stop()`, which
will make the loop stop after the current event iteration completes.
"""
raise NotImplementedError()
@@ -266,7 +296,7 @@ class IOLoop(Configurable):
def stop(self):
"""Stop the I/O loop.
If the event loop is not currently running, the next call to start()
If the event loop is not currently running, the next call to `start()`
will return immediately.
To use asynchronous methods from otherwise-synchronous code (such as
@@ -276,23 +306,71 @@ class IOLoop(Configurable):
async_method(ioloop=ioloop, callback=ioloop.stop)
ioloop.start()
ioloop.start() will return after async_method has run its callback,
whether that callback was invoked before or after ioloop.start.
``ioloop.start()`` will return after ``async_method`` has run
its callback, whether that callback was invoked before or
after ``ioloop.start``.
Note that even after `stop` has been called, the IOLoop is not
Note that even after `stop` has been called, the `IOLoop` is not
completely stopped until `IOLoop.start` has also returned.
Some work that was scheduled before the call to `stop` may still
be run before the IOLoop shuts down.
be run before the `IOLoop` shuts down.
"""
raise NotImplementedError()
def run_sync(self, func, timeout=None):
"""Starts the `IOLoop`, runs the given function, and stops the loop.
If the function returns a `.Future`, the `IOLoop` will run
until the future is resolved. If it raises an exception, the
`IOLoop` will stop and the exception will be re-raised to the
caller.
The keyword-only argument ``timeout`` may be used to set
a maximum duration for the function. If the timeout expires,
a `TimeoutError` is raised.
This method is useful in conjunction with `tornado.gen.coroutine`
to allow asynchronous calls in a ``main()`` function::
@gen.coroutine
def main():
# do stuff...
if __name__ == '__main__':
IOLoop.instance().run_sync(main)
"""
future_cell = [None]
def run():
try:
result = func()
except Exception:
future_cell[0] = TracebackFuture()
future_cell[0].set_exc_info(sys.exc_info())
else:
if isinstance(result, Future):
future_cell[0] = result
else:
future_cell[0] = Future()
future_cell[0].set_result(result)
self.add_future(future_cell[0], lambda future: self.stop())
self.add_callback(run)
if timeout is not None:
timeout_handle = self.add_timeout(self.time() + timeout, self.stop)
self.start()
if timeout is not None:
self.remove_timeout(timeout_handle)
if not future_cell[0].done():
raise TimeoutError('Operation timed out after %s seconds' % timeout)
return future_cell[0].result()
def time(self):
"""Returns the current time according to the IOLoop's clock.
"""Returns the current time according to the `IOLoop`'s clock.
The return value is a floating-point number relative to an
unspecified time in the past.
By default, the IOLoop's time function is `time.time`. However,
By default, the `IOLoop`'s time function is `time.time`. However,
it may be configured to use e.g. `time.monotonic` instead.
Calls to `add_timeout` that pass a number instead of a
`datetime.timedelta` should use this function to compute the
@@ -302,24 +380,26 @@ class IOLoop(Configurable):
return time.time()
def add_timeout(self, deadline, callback):
"""Calls the given callback at the time deadline from the I/O loop.
"""Runs the ``callback`` at the time ``deadline`` from the I/O loop.
Returns a handle that may be passed to remove_timeout to cancel.
Returns an opaque handle that may be passed to
`remove_timeout` to cancel.
``deadline`` may be a number denoting a time relative to
`IOLoop.time`, or a ``datetime.timedelta`` object for a
deadline relative to the current time.
``deadline`` may be a number denoting a time (on the same
scale as `IOLoop.time`, normally `time.time`), or a
`datetime.timedelta` object for a deadline relative to the
current time.
Note that it is not safe to call `add_timeout` from other threads.
Instead, you must use `add_callback` to transfer control to the
IOLoop's thread, and then call `add_timeout` from there.
`IOLoop`'s thread, and then call `add_timeout` from there.
"""
raise NotImplementedError()
def remove_timeout(self, timeout):
"""Cancels a pending timeout.
The argument is a handle as returned by add_timeout. It is
The argument is a handle as returned by `add_timeout`. It is
safe to call `remove_timeout` even if the callback has already
been run.
"""
@@ -329,11 +409,11 @@ class IOLoop(Configurable):
"""Calls the given callback on the next I/O loop iteration.
It is safe to call this method from any thread at any time,
except from a signal handler. Note that this is the *only*
method in IOLoop that makes this thread-safety guarantee; all
other interaction with the IOLoop must be done from that
IOLoop's thread. add_callback() may be used to transfer
control from other threads to the IOLoop's thread.
except from a signal handler. Note that this is the **only**
method in `IOLoop` that makes this thread-safety guarantee; all
other interaction with the `IOLoop` must be done from that
`IOLoop`'s thread. `add_callback()` may be used to transfer
control from other threads to the `IOLoop`'s thread.
To add a callback from a signal handler, see
`add_callback_from_signal`.
@@ -347,22 +427,19 @@ class IOLoop(Configurable):
otherwise.
Callbacks added with this method will be run without any
stack_context, to avoid picking up the context of the function
`.stack_context`, to avoid picking up the context of the function
that was interrupted by the signal.
"""
raise NotImplementedError()
if futures is not None:
_FUTURE_TYPES = (futures.Future, DummyFuture)
else:
_FUTURE_TYPES = DummyFuture
def add_future(self, future, callback):
"""Schedules a callback on the IOLoop when the given future is finished.
"""Schedules a callback on the ``IOLoop`` when the given
`.Future` is finished.
The callback is invoked with one argument, the future.
The callback is invoked with one argument, the
`.Future`.
"""
assert isinstance(future, IOLoop._FUTURE_TYPES)
assert isinstance(future, Future)
callback = stack_context.wrap(callback)
future.add_done_callback(
lambda future: self.add_callback(callback, future))
@@ -378,14 +455,14 @@ class IOLoop(Configurable):
self.handle_callback_exception(callback)
def handle_callback_exception(self, callback):
"""This method is called whenever a callback run by the IOLoop
"""This method is called whenever a callback run by the `IOLoop`
throws an exception.
By default simply logs the exception as an error. Subclasses
may override this method to customize reporting of exceptions.
The exception itself is not passed explicitly, but is available
in sys.exc_info.
in `sys.exc_info`.
"""
app_log.error("Exception in callback %r", callback, exc_info=True)
@@ -428,7 +505,11 @@ class PollIOLoop(IOLoop):
if all_fds:
for fd in self._handlers.keys():
try:
os.close(fd)
close_method = getattr(fd, 'close', None)
if close_method is not None:
close_method()
else:
os.close(fd)
except Exception:
gen_log.debug("error closing fd %s", fd, exc_info=True)
self._waker.close()
@@ -684,16 +765,16 @@ class _Timeout(object):
class PeriodicCallback(object):
"""Schedules the given callback to be called periodically.
The callback is called every callback_time milliseconds.
The callback is called every ``callback_time`` milliseconds.
`start` must be called after the PeriodicCallback is created.
`start` must be called after the `PeriodicCallback` is created.
"""
def __init__(self, callback, callback_time, io_loop=None):
self.callback = callback
if callback_time <= 0:
raise ValueError("Periodic callback must have a positive callback_time")
self.callback_time = callback_time
self.io_loop = io_loop or IOLoop.instance()
self.io_loop = io_loop or IOLoop.current()
self._running = False
self._timeout = None
libs/tornado/iostream.py
+57 -45
View File
@@ -58,7 +58,7 @@ class BaseIOStream(object):
All of the methods take callbacks (since writing and reading are
non-blocking and asynchronous).
When a stream is closed due to an error, the IOStream's `error`
When a stream is closed due to an error, the IOStream's ``error``
attribute contains the exception object.
Subclasses must implement `fileno`, `close_fd`, `write_to_fd`,
@@ -66,7 +66,7 @@ class BaseIOStream(object):
"""
def __init__(self, io_loop=None, max_buffer_size=104857600,
read_chunk_size=4096):
self.io_loop = io_loop or ioloop.IOLoop.instance()
self.io_loop = io_loop or ioloop.IOLoop.current()
self.max_buffer_size = max_buffer_size
self.read_chunk_size = read_chunk_size
self.error = None
@@ -110,16 +110,17 @@ class BaseIOStream(object):
def read_from_fd(self):
"""Attempts to read from the underlying file.
Returns ``None`` if there was nothing to read (the socket returned
EWOULDBLOCK or equivalent), otherwise returns the data. When possible,
should return no more than ``self.read_chunk_size`` bytes at a time.
Returns ``None`` if there was nothing to read (the socket
returned `~errno.EWOULDBLOCK` or equivalent), otherwise
returns the data. When possible, should return no more than
``self.read_chunk_size`` bytes at a time.
"""
raise NotImplementedError()
def get_fd_error(self):
"""Returns information about any error on the underlying file.
This method is called after the IOLoop has signaled an error on the
This method is called after the `.IOLoop` has signaled an error on the
file descriptor, and should return an Exception (such as `socket.error`
with additional information, or None if no such information is
available.
@@ -127,23 +128,32 @@ class BaseIOStream(object):
return None
def read_until_regex(self, regex, callback):
"""Call callback when we read the given regex pattern."""
"""Run ``callback`` when we read the given regex pattern.
The callback will get the data read (including the data that
matched the regex and anything that came before it) as an argument.
"""
self._set_read_callback(callback)
self._read_regex = re.compile(regex)
self._try_inline_read()
def read_until(self, delimiter, callback):
"""Call callback when we read the given delimiter."""
"""Run ``callback`` when we read the given delimiter.
The callback will get the data read (including the delimiter)
as an argument.
"""
self._set_read_callback(callback)
self._read_delimiter = delimiter
self._try_inline_read()
def read_bytes(self, num_bytes, callback, streaming_callback=None):
"""Call callback when we read the given number of bytes.
"""Run callback when we read the given number of bytes.
If a ``streaming_callback`` is given, it will be called with chunks
of data as they become available, and the argument to the final
``callback`` will be empty.
``callback`` will be empty. Otherwise, the ``callback`` gets
the data as an argument.
"""
self._set_read_callback(callback)
assert isinstance(num_bytes, numbers.Integral)
@@ -156,7 +166,8 @@ class BaseIOStream(object):
If a ``streaming_callback`` is given, it will be called with chunks
of data as they become available, and the argument to the final
``callback`` will be empty.
``callback`` will be empty. Otherwise, the ``callback`` gets the
data as an argument.
Subject to ``max_buffer_size`` limit from `IOStream` constructor if
a ``streaming_callback`` is not used.
@@ -174,12 +185,12 @@ class BaseIOStream(object):
return
self._read_until_close = True
self._streaming_callback = stack_context.wrap(streaming_callback)
self._add_io_state(self.io_loop.READ)
self._try_inline_read()
def write(self, data, callback=None):
"""Write the given data to this stream.
If callback is given, we call it when all of the buffered write
If ``callback`` is given, we call it when all of the buffered write
data has been successfully written to the stream. If there was
previously buffered write data and an old write callback, that
callback is simply overwritten with this new callback.
@@ -213,7 +224,7 @@ class BaseIOStream(object):
"""Close this stream.
If ``exc_info`` is true, set the ``error`` attribute to the current
exception from `sys.exc_info()` (or if ``exc_info`` is a tuple,
exception from `sys.exc_info` (or if ``exc_info`` is a tuple,
use that instead of `sys.exc_info`).
"""
if not self.closed():
@@ -573,44 +584,45 @@ class BaseIOStream(object):
class IOStream(BaseIOStream):
r"""Socket-based IOStream implementation.
r"""Socket-based `IOStream` implementation.
This class supports the read and write methods from `BaseIOStream`
plus a `connect` method.
The socket parameter may either be connected or unconnected. For
server operations the socket is the result of calling socket.accept().
For client operations the socket is created with socket.socket(),
and may either be connected before passing it to the IOStream or
connected with IOStream.connect.
The ``socket`` parameter may either be connected or unconnected.
For server operations the socket is the result of calling
`socket.accept <socket.socket.accept>`. For client operations the
socket is created with `socket.socket`, and may either be
connected before passing it to the `IOStream` or connected with
`IOStream.connect`.
A very simple (and broken) HTTP client using this class::
from tornado import ioloop
from tornado import iostream
import tornado.ioloop
import tornado.iostream
import socket
def send_request():
stream.write("GET / HTTP/1.0\r\nHost: friendfeed.com\r\n\r\n")
stream.read_until("\r\n\r\n", on_headers)
stream.write(b"GET / HTTP/1.0\r\nHost: friendfeed.com\r\n\r\n")
stream.read_until(b"\r\n\r\n", on_headers)
def on_headers(data):
headers = {}
for line in data.split("\r\n"):
parts = line.split(":")
for line in data.split(b"\r\n"):
parts = line.split(b":")
if len(parts) == 2:
headers[parts[0].strip()] = parts[1].strip()
stream.read_bytes(int(headers["Content-Length"]), on_body)
stream.read_bytes(int(headers[b"Content-Length"]), on_body)
def on_body(data):
print data
stream.close()
ioloop.IOLoop.instance().stop()
tornado.ioloop.IOLoop.instance().stop()
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
stream = iostream.IOStream(s)
stream = tornado.iostream.IOStream(s)
stream.connect(("friendfeed.com", 80), send_request)
ioloop.IOLoop.instance().start()
tornado.ioloop.IOLoop.instance().start()
"""
def __init__(self, socket, *args, **kwargs):
self.socket = socket
@@ -650,20 +662,20 @@ class IOStream(BaseIOStream):
May only be called if the socket passed to the constructor was
not previously connected. The address parameter is in the
same format as for socket.connect, i.e. a (host, port) tuple.
If callback is specified, it will be called when the
connection is completed.
same format as for `socket.connect <socket.socket.connect>`,
i.e. a ``(host, port)`` tuple. If ``callback`` is specified,
it will be called when the connection is completed.
If specified, the ``server_hostname`` parameter will be used
in SSL connections for certificate validation (if requested in
the ``ssl_options``) and SNI (if supported; requires
Python 3.2+).
Note that it is safe to call IOStream.write while the
connection is pending, in which case the data will be written
as soon as the connection is ready. Calling IOStream read
methods before the socket is connected works on some platforms
but is non-portable.
Note that it is safe to call `IOStream.write
<BaseIOStream.write>` while the connection is pending, in
which case the data will be written as soon as the connection
is ready. Calling `IOStream` read methods before the socket is
connected works on some platforms but is non-portable.
"""
self._connecting = True
try:
@@ -711,13 +723,11 @@ class SSLIOStream(IOStream):
ssl.wrap_socket(sock, do_handshake_on_connect=False, **kwargs)
before constructing the SSLIOStream. Unconnected sockets will be
wrapped when IOStream.connect is finished.
before constructing the `SSLIOStream`. Unconnected sockets will be
wrapped when `IOStream.connect` is finished.
"""
def __init__(self, *args, **kwargs):
"""Creates an SSLIOStream.
The ``ssl_options`` keyword argument may either be a dictionary
"""The ``ssl_options`` keyword argument may either be a dictionary
of keywords arguments for `ssl.wrap_socket`, or an `ssl.SSLContext`
object.
"""
@@ -863,10 +873,12 @@ class SSLIOStream(IOStream):
class PipeIOStream(BaseIOStream):
"""Pipe-based IOStream implementation.
"""Pipe-based `IOStream` implementation.
The constructor takes an integer file descriptor (such as one returned
by `os.pipe`) rather than an open file object.
by `os.pipe`) rather than an open file object. Pipes are generally
one-way, so a `PipeIOStream` can be used for reading or writing but not
both.
"""
def __init__(self, fd, *args, **kwargs):
self.fd = fd
libs/tornado/locale.py
+42 -38
View File
@@ -1,5 +1,5 @@
#!/usr/bin/env python
#
# -*- coding: utf-8 -*-
# Copyright 2009 Facebook
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -18,25 +18,25 @@
To load a locale and generate a translated string::
user_locale = locale.get("es_LA")
user_locale = tornado.locale.get("es_LA")
print user_locale.translate("Sign out")
locale.get() returns the closest matching locale, not necessarily the
`tornado.locale.get()` returns the closest matching locale, not necessarily the
specific locale you requested. You can support pluralization with
additional arguments to translate(), e.g.::
additional arguments to `~Locale.translate()`, e.g.::
people = [...]
message = user_locale.translate(
"%(list)s is online", "%(list)s are online", len(people))
print message % {"list": user_locale.list(people)}
The first string is chosen if len(people) == 1, otherwise the second
The first string is chosen if ``len(people) == 1``, otherwise the second
string is chosen.
Applications should call one of load_translations (which uses a simple
CSV format) or load_gettext_translations (which uses the .mo format
supported by gettext and related tools). If neither method is called,
the locale.translate method will simply return the original string.
Applications should call one of `load_translations` (which uses a simple
CSV format) or `load_gettext_translations` (which uses the ``.mo`` format
supported by `gettext` and related tools). If neither method is called,
the `Locale.translate` method will simply return the original string.
"""
from __future__ import absolute_import, division, print_function, with_statement
@@ -63,15 +63,15 @@ def get(*locale_codes):
or a loose match for the code (e.g., "en" for "en_US"), we return
the locale. Otherwise we move to the next code in the list.
By default we return en_US if no translations are found for any of
By default we return ``en_US`` if no translations are found for any of
the specified locales. You can change the default locale with
set_default_locale() below.
`set_default_locale()`.
"""
return Locale.get_closest(*locale_codes)
def set_default_locale(code):
"""Sets the default locale, used in get_closest_locale().
"""Sets the default locale.
The default locale is assumed to be the language used for all strings
in the system. The translations loaded from disk are mappings from
@@ -85,32 +85,32 @@ def set_default_locale(code):
def load_translations(directory):
u("""Loads translations from CSV files in a directory.
"""Loads translations from CSV files in a directory.
Translations are strings with optional Python-style named placeholders
(e.g., "My name is %(name)s") and their associated translations.
(e.g., ``My name is %(name)s``) and their associated translations.
The directory should have translation files of the form LOCALE.csv,
e.g. es_GT.csv. The CSV files should have two or three columns: string,
The directory should have translation files of the form ``LOCALE.csv``,
e.g. ``es_GT.csv``. The CSV files should have two or three columns: string,
translation, and an optional plural indicator. Plural indicators should
be one of "plural" or "singular". A given string can have both singular
and plural forms. For example "%(name)s liked this" may have a
and plural forms. For example ``%(name)s liked this`` may have a
different verb conjugation depending on whether %(name)s is one
name or a list of names. There should be two rows in the CSV file for
that string, one with plural indicator "singular", and one "plural".
For strings with no verbs that would change on translation, simply
use "unknown" or the empty string (or don't include the column at all).
The file is read using the csv module in the default "excel" dialect.
The file is read using the `csv` module in the default "excel" dialect.
In this format there should not be spaces after the commas.
Example translation es_LA.csv:
Example translation ``es_LA.csv``::
"I love you","Te amo"
"%(name)s liked this","A %(name)s les gust\u00f3 esto","plural"
"%(name)s liked this","A %(name)s le gust\u00f3 esto","singular"
"%(name)s liked this","A %(name)s les gustó esto","plural"
"%(name)s liked this","A %(name)s le gustó esto","singular"
""")
"""
global _translations
global _supported_locales
_translations = {}
@@ -151,22 +151,25 @@ def load_translations(directory):
def load_gettext_translations(directory, domain):
"""Loads translations from gettext's locale tree
"""Loads translations from `gettext`'s locale tree
Locale tree is similar to system's /usr/share/locale, like:
Locale tree is similar to system's ``/usr/share/locale``, like::
{directory}/{lang}/LC_MESSAGES/{domain}.mo
{directory}/{lang}/LC_MESSAGES/{domain}.mo
Three steps are required to have you app translated:
1. Generate POT translation file
xgettext --language=Python --keyword=_:1,2 -d cyclone file1.py file2.html etc
1. Generate POT translation file::
2. Merge against existing POT file:
msgmerge old.po cyclone.po > new.po
xgettext --language=Python --keyword=_:1,2 -d mydomain file1.py file2.html etc
3. Compile:
msgfmt cyclone.po -o {directory}/pt_BR/LC_MESSAGES/cyclone.mo
2. Merge against existing POT file::
msgmerge old.po mydomain.po > new.po
3. Compile::
msgfmt mydomain.po -o {directory}/pt_BR/LC_MESSAGES/mydomain.mo
"""
import gettext
global _translations
@@ -262,9 +265,10 @@ class Locale(object):
def translate(self, message, plural_message=None, count=None):
"""Returns the translation for the given message for this locale.
If plural_message is given, you must also provide count. We return
plural_message when count != 1, and we return the singular form
for the given message when count == 1.
If ``plural_message`` is given, you must also provide
``count``. We return ``plural_message`` when ``count != 1``,
and we return the singular form for the given message when
``count == 1``.
"""
raise NotImplementedError()
@@ -273,10 +277,10 @@ class Locale(object):
"""Formats the given date (which should be GMT).
By default, we return a relative time (e.g., "2 minutes ago"). You
can return an absolute date string with relative=False.
can return an absolute date string with ``relative=False``.
You can force a full format date ("July 10, 1980") with
full_format=True.
``full_format=True``.
This method is primarily intended for dates in the past.
For dates in the future, we fall back to full format.
@@ -360,7 +364,7 @@ class Locale(object):
"""Formats the given date as a day of week.
Example: "Monday, January 22". You can remove the day of week with
dow=False.
``dow=False``.
"""
local_date = date - datetime.timedelta(minutes=gmt_offset)
_ = self.translate
@@ -421,7 +425,7 @@ class CSVLocale(Locale):
class GettextLocale(Locale):
"""Locale implementation using the gettext module."""
"""Locale implementation using the `gettext` module."""
def __init__(self, code, translations):
try:
# python 2
libs/tornado/netutil.py
+90 -33
View File
@@ -41,14 +41,14 @@ def bind_sockets(port, address=None, family=socket.AF_UNSPEC, backlog=128, flags
Address may be either an IP address or hostname. If it's a hostname,
the server will listen on all IP addresses associated with the
name. Address may be an empty string or None to listen on all
available interfaces. Family may be set to either socket.AF_INET
or socket.AF_INET6 to restrict to ipv4 or ipv6 addresses, otherwise
available interfaces. Family may be set to either `socket.AF_INET`
or `socket.AF_INET6` to restrict to IPv4 or IPv6 addresses, otherwise
both will be used if available.
The ``backlog`` argument has the same meaning as for
``socket.listen()``.
`socket.listen() <socket.socket.listen>`.
``flags`` is a bitmask of AI_* flags to ``getaddrinfo``, like
``flags`` is a bitmask of AI_* flags to `~socket.getaddrinfo`, like
``socket.AI_PASSIVE | socket.AI_NUMERICHOST``.
"""
sockets = []
@@ -119,16 +119,16 @@ if hasattr(socket, 'AF_UNIX'):
def add_accept_handler(sock, callback, io_loop=None):
"""Adds an ``IOLoop`` event handler to accept new connections on ``sock``.
"""Adds an `.IOLoop` event handler to accept new connections on ``sock``.
When a connection is accepted, ``callback(connection, address)`` will
be run (``connection`` is a socket object, and ``address`` is the
address of the other end of the connection). Note that this signature
is different from the ``callback(fd, events)`` signature used for
``IOLoop`` handlers.
`.IOLoop` handlers.
"""
if io_loop is None:
io_loop = IOLoop.instance()
io_loop = IOLoop.current()
def accept_handler(fd, events):
while True:
@@ -142,7 +142,41 @@ def add_accept_handler(sock, callback, io_loop=None):
io_loop.add_handler(sock.fileno(), accept_handler, IOLoop.READ)
def is_valid_ip(ip):
"""Returns true if the given string is a well-formed IP address.
Supports IPv4 and IPv6.
"""
try:
res = socket.getaddrinfo(ip, 0, socket.AF_UNSPEC,
socket.SOCK_STREAM,
0, socket.AI_NUMERICHOST)
return bool(res)
except socket.gaierror as e:
if e.args[0] == socket.EAI_NONAME:
return False
raise
return True
class Resolver(Configurable):
"""Configurable asynchronous DNS resolver interface.
By default, a blocking implementation is used (which simply calls
`socket.getaddrinfo`). An alternative implementation can be
chosen with the `Resolver.configure <.Configurable.configure>`
class method::
Resolver.configure('tornado.netutil.ThreadedResolver')
The implementations of this interface included with Tornado are
* `tornado.netutil.BlockingResolver`
* `tornado.netutil.ThreadedResolver`
* `tornado.netutil.OverrideResolver`
* `tornado.platform.twisted.TwistedResolver`
* `tornado.platform.caresresolver.CaresResolver`
"""
@classmethod
def configurable_base(cls):
return Resolver
@@ -151,40 +185,64 @@ class Resolver(Configurable):
def configurable_default(cls):
return BlockingResolver
def getaddrinfo(self, *args, **kwargs):
def resolve(self, host, port, family=socket.AF_UNSPEC, callback=None):
"""Resolves an address.
The arguments to this function are the same as to
`socket.getaddrinfo`, with the addition of an optional
keyword-only ``callback`` argument.
The ``host`` argument is a string which may be a hostname or a
literal IP address.
Returns a `Future` whose result is the same as the return
value of `socket.getaddrinfo`. If a callback is passed,
it will be run with the `Future` as an argument when it
is complete.
Returns a `.Future` whose result is a list of (family,
address) pairs, where address is a tuple suitable to pass to
`socket.connect <socket.socket.connect>` (i.e. a ``(host,
port)`` pair for IPv4; additional fields may be present for
IPv6). If a ``callback`` is passed, it will be run with the
result as an argument when it is complete.
"""
raise NotImplementedError()
class ExecutorResolver(Resolver):
def initialize(self, io_loop=None, executor=None):
self.io_loop = io_loop or IOLoop.instance()
self.io_loop = io_loop or IOLoop.current()
self.executor = executor or dummy_executor
@run_on_executor
def getaddrinfo(self, *args, **kwargs):
return socket.getaddrinfo(*args, **kwargs)
def resolve(self, host, port, family=socket.AF_UNSPEC):
addrinfo = socket.getaddrinfo(host, port, family)
results = []
for family, socktype, proto, canonname, address in addrinfo:
results.append((family, address))
return results
class BlockingResolver(ExecutorResolver):
"""Default `Resolver` implementation, using `socket.getaddrinfo`.
The `.IOLoop` will be blocked during the resolution, although the
callback will not be run until the next `.IOLoop` iteration.
"""
def initialize(self, io_loop=None):
super(BlockingResolver, self).initialize(io_loop=io_loop)
class ThreadedResolver(ExecutorResolver):
"""Multithreaded non-blocking `Resolver` implementation.
Requires the `concurrent.futures` package to be installed
(available in the standard library since Python 3.2,
installable with ``pip install futures`` in older versions).
The thread pool size can be configured with::
Resolver.configure('tornado.netutil.ThreadedResolver',
num_threads=10)
"""
def initialize(self, io_loop=None, num_threads=10):
from concurrent.futures import ThreadPoolExecutor
super(ThreadedResolver, self).initialize(
io_loop=io_loop, executor=ThreadPoolExecutor(num_threads))
class OverrideResolver(Resolver):
"""Wraps a resolver with a mapping of overrides.
@@ -197,13 +255,12 @@ class OverrideResolver(Resolver):
self.resolver = resolver
self.mapping = mapping
def getaddrinfo(self, host, port, *args, **kwargs):
def resolve(self, host, port, *args, **kwargs):
if (host, port) in self.mapping:
host, port = self.mapping[(host, port)]
elif host in self.mapping:
host = self.mapping[host]
return self.resolver.getaddrinfo(host, port, *args, **kwargs)
return self.resolver.resolve(host, port, *args, **kwargs)
# These are the keyword arguments to ssl.wrap_socket that must be translated
@@ -212,20 +269,22 @@ class OverrideResolver(Resolver):
_SSL_CONTEXT_KEYWORDS = frozenset(['ssl_version', 'certfile', 'keyfile',
'cert_reqs', 'ca_certs', 'ciphers'])
def ssl_options_to_context(ssl_options):
"""Try to Convert an ssl_options dictionary to an SSLContext object.
"""Try to convert an ``ssl_options`` dictionary to an
`~ssl.SSLContext` object.
The ``ssl_options`` dictionary contains keywords to be passed to
`ssl.wrap_sockets`. In Python 3.2+, `ssl.SSLContext` objects can
`ssl.wrap_socket`. In Python 3.2+, `ssl.SSLContext` objects can
be used instead. This function converts the dict form to its
`SSLContext` equivalent, and may be used when a component which
accepts both forms needs to upgrade to the `SSLContext` version
`~ssl.SSLContext` equivalent, and may be used when a component which
accepts both forms needs to upgrade to the `~ssl.SSLContext` version
to use features like SNI or NPN.
"""
if isinstance(ssl_options, dict):
assert all(k in _SSL_CONTEXT_KEYWORDS for k in ssl_options), ssl_options
if (not hasattr(ssl, 'SSLContext') or
isinstance(ssl_options, ssl.SSLContext)):
isinstance(ssl_options, ssl.SSLContext)):
return ssl_options
context = ssl.SSLContext(
ssl_options.get('ssl_version', ssl.PROTOCOL_SSLv23))
@@ -241,12 +300,12 @@ def ssl_options_to_context(ssl_options):
def ssl_wrap_socket(socket, ssl_options, server_hostname=None, **kwargs):
"""Returns an `ssl.SSLSocket` wrapping the given socket.
"""Returns an ``ssl.SSLSocket`` wrapping the given socket.
``ssl_options`` may be either a dictionary (as accepted by
`ssl_options_to_context) or an `ssl.SSLContext` object.
Additional keyword arguments are passed to `wrap_socket`
(either the `SSLContext` method or the `ssl` module function
`ssl_options_to_context`) or an `ssl.SSLContext` object.
Additional keyword arguments are passed to ``wrap_socket``
(either the `~ssl.SSLContext` method or the `ssl` module function
as appropriate).
"""
context = ssl_options_to_context(ssl_options)
@@ -262,7 +321,7 @@ def ssl_wrap_socket(socket, ssl_options, server_hostname=None, **kwargs):
else:
return ssl.wrap_socket(socket, **dict(context, **kwargs))
if hasattr(ssl, 'match_hostname'): # python 3.2+
if hasattr(ssl, 'match_hostname') and hasattr(ssl, 'CertificateError'): # python 3.2+
ssl_match_hostname = ssl.match_hostname
SSLCertificateError = ssl.CertificateError
else:
@@ -272,7 +331,6 @@ else:
class SSLCertificateError(ValueError):
pass
def _dnsname_to_pat(dn):
pats = []
for frag in dn.split(r'.'):
@@ -286,7 +344,6 @@ else:
pats.append(frag.replace(r'\*', '[^.]*'))
return re.compile(r'\A' + r'\.'.join(pats) + r'\Z', re.IGNORECASE)
def ssl_match_hostname(cert, hostname):
"""Verify that *cert* (in decoded format as returned by
SSLSocket.getpeercert()) matches the *hostname*. RFC 2818 rules
libs/tornado/options.py
+35 -29
View File
@@ -29,27 +29,28 @@ option namespace, e.g.::
db = database.Connection(options.mysql_host)
...
The main() method of your application does not need to be aware of all of
The ``main()`` method of your application does not need to be aware of all of
the options used throughout your program; they are all automatically loaded
when the modules are loaded. However, all modules that define options
must have been imported before the command line is parsed.
Your main() method can parse the command line or parse a config file with
Your ``main()`` method can parse the command line or parse a config file with
either::
tornado.options.parse_command_line()
# or
tornado.options.parse_config_file("/etc/server.conf")
Command line formats are what you would expect ("--myoption=myvalue").
Command line formats are what you would expect (``--myoption=myvalue``).
Config files are just Python files. Global names become options, e.g.::
myoption = "myvalue"
myotheroption = "myothervalue"
We support datetimes, timedeltas, ints, and floats (just pass a 'type'
kwarg to define). We also accept multi-value options. See the documentation
for define() below.
We support `datetimes <datetime.datetime>`, `timedeltas
<datetime.timedelta>`, ints, and floats (just pass a ``type`` kwarg to
`define`). We also accept multi-value options. See the documentation for
`define()` below.
`tornado.options.options` is a singleton instance of `OptionParser`, and
the top-level functions in this module (`define`, `parse_command_line`, etc)
@@ -68,7 +69,7 @@ import textwrap
from tornado.escape import _unicode
from tornado.log import define_logging_options
from tornado import stack_context
from tornado.util import basestring_type
from tornado.util import basestring_type, exec_in
class Error(Exception):
@@ -103,28 +104,29 @@ class OptionParser(object):
multiple=False, group=None, callback=None):
"""Defines a new command line option.
If type is given (one of str, float, int, datetime, or timedelta)
or can be inferred from the default, we parse the command line
arguments based on the given type. If multiple is True, we accept
If ``type`` is given (one of str, float, int, datetime, or timedelta)
or can be inferred from the ``default``, we parse the command line
arguments based on the given type. If ``multiple`` is True, we accept
comma-separated values, and the option value is always a list.
For multi-value integers, we also accept the syntax x:y, which
turns into range(x, y) - very useful for long integer ranges.
For multi-value integers, we also accept the syntax ``x:y``, which
turns into ``range(x, y)`` - very useful for long integer ranges.
help and metavar are used to construct the automatically generated
command line help string. The help message is formatted like::
``help`` and ``metavar`` are used to construct the
automatically generated command line help string. The help
message is formatted like::
--name=METAVAR help string
group is used to group the defined options in logical
``group`` is used to group the defined options in logical
groups. By default, command line options are grouped by the
file in which they are defined.
Command line option names must be unique globally. They can be parsed
from the command line with parse_command_line() or parsed from a
config file with parse_config_file.
from the command line with `parse_command_line` or parsed from a
config file with `parse_config_file`.
If a callback is given, it will be run with the new value whenever
If a ``callback`` is given, it will be run with the new value whenever
the option is changed. This can be used to combine command-line
and file-based options::
@@ -159,9 +161,11 @@ class OptionParser(object):
callback=callback)
def parse_command_line(self, args=None, final=True):
"""Parses all options given on the command line (defaults to sys.argv).
"""Parses all options given on the command line (defaults to
`sys.argv`).
Note that args[0] is ignored since it is the program name in sys.argv.
Note that ``args[0]`` is ignored since it is the program name
in `sys.argv`.
We return a list of all arguments that are not parsed as options.
@@ -207,7 +211,8 @@ class OptionParser(object):
from multiple sources.
"""
config = {}
execfile(path, config, config)
with open(path) as f:
exec_in(f.read(), config, config)
for name in config:
if name in self._options:
self._options[name].set(config[name])
@@ -258,15 +263,16 @@ class OptionParser(object):
callback()
def mockable(self):
"""Returns a wrapper around self that is compatible with `mock.patch`.
"""Returns a wrapper around self that is compatible with
`mock.patch <unittest.mock.patch>`.
The `mock.patch` function (included in the standard library
`unittest.mock` package since Python 3.3, or in the
third-party `mock` package for older versions of Python) is
incompatible with objects like ``options`` that override
``__getattr__`` and ``__setattr__``. This function returns an
object that can be used with `mock.patch.object` to modify
option values::
The `mock.patch <unittest.mock.patch>` function (included in
the standard library `unittest.mock` package since Python 3.3,
or in the third-party ``mock`` package for older versions of
Python) is incompatible with objects like ``options`` that
override ``__getattr__`` and ``__setattr__``. This function
returns an object that can be used with `mock.patch.object
<unittest.mock.patch.object>` to modify option values::
with mock.patch.object(options.mockable(), 'name', value):
assert options.name == value
libs/tornado/platform/caresresolver.py
Executable
+75
View File
@@ -0,0 +1,75 @@
import pycares
import socket
from tornado import gen
from tornado.ioloop import IOLoop
from tornado.netutil import Resolver, is_valid_ip
class CaresResolver(Resolver):
"""Name resolver based on the c-ares library.
This is a non-blocking and non-threaded resolver. It may not produce
the same results as the system resolver, but can be used for non-blocking
resolution when threads cannot be used.
c-ares fails to resolve some names when ``family`` is ``AF_UNSPEC``,
so it is only recommended for use in ``AF_INET`` (i.e. IPv4). This is
the default for ``tornado.simple_httpclient``, but other libraries
may default to ``AF_UNSPEC``.
"""
def initialize(self, io_loop=None):
self.io_loop = io_loop or IOLoop.current()
self.channel = pycares.Channel(sock_state_cb=self._sock_state_cb)
self.fds = {}
def _sock_state_cb(self, fd, readable, writable):
state = ((IOLoop.READ if readable else 0) |
(IOLoop.WRITE if writable else 0))
if not state:
self.io_loop.remove_handler(fd)
del self.fds[fd]
elif fd in self.fds:
self.io_loop.update_handler(fd, state)
self.fds[fd] = state
else:
self.io_loop.add_handler(fd, self._handle_events, state)
self.fds[fd] = state
def _handle_events(self, fd, events):
read_fd = pycares.ARES_SOCKET_BAD
write_fd = pycares.ARES_SOCKET_BAD
if events & IOLoop.READ:
read_fd = fd
if events & IOLoop.WRITE:
write_fd = fd
self.channel.process_fd(read_fd, write_fd)
@gen.coroutine
def resolve(self, host, port, family=0):
if is_valid_ip(host):
addresses = [host]
else:
# gethostbyname doesn't take callback as a kwarg
self.channel.gethostbyname(host, family, (yield gen.Callback(1)))
callback_args = yield gen.Wait(1)
assert isinstance(callback_args, gen.Arguments)
assert not callback_args.kwargs
result, error = callback_args.args
if error:
raise Exception('C-Ares returned error %s: %s while resolving %s' %
(error, pycares.errno.strerror(error), host))
addresses = result.addresses
addrinfo = []
for address in addresses:
if '.' in address:
address_family = socket.AF_INET
elif ':' in address:
address_family = socket.AF_INET6
else:
address_family = socket.AF_UNSPEC
if family != socket.AF_UNSPEC and family != address_family:
raise Exception('Requested socket family %d but got %d' %
(family, address_family))
addrinfo.append((address_family, (address, port)))
raise gen.Return(addrinfo)
libs/tornado/platform/twisted.py
+87 -15
View File
@@ -22,6 +22,8 @@ This module lets you run applications and libraries written for
Twisted in a Tornado application. It can be used in two modes,
depending on which library's underlying event loop you want to use.
This module has been tested with Twisted versions 11.0.0 and newer.
Twisted on Tornado
------------------
@@ -60,26 +62,33 @@ reactor. Recommended usage::
reactor.run()
`TwistedIOLoop` always uses the global Twisted reactor.
This module has been tested with Twisted versions 11.0.0 and newer.
"""
from __future__ import absolute_import, division, print_function, with_statement
import functools
import datetime
import functools
import socket
import twisted.internet.abstract
from twisted.internet.posixbase import PosixReactorBase
from twisted.internet.interfaces import \
IReactorFDSet, IDelayedCall, IReactorTime, IReadDescriptor, IWriteDescriptor
from twisted.python import failure, log
from twisted.internet import error
import twisted.names.cache
import twisted.names.client
import twisted.names.hosts
import twisted.names.resolve
from zope.interface import implementer
import tornado
from tornado.concurrent import return_future
from tornado.escape import utf8
from tornado import gen
import tornado.ioloop
from tornado.log import app_log
from tornado.netutil import Resolver
from tornado.stack_context import NullContext, wrap
from tornado.ioloop import IOLoop
@@ -140,7 +149,7 @@ class TornadoReactor(PosixReactorBase):
"""
def __init__(self, io_loop=None):
if not io_loop:
io_loop = tornado.ioloop.IOLoop.instance()
io_loop = tornado.ioloop.IOLoop.current()
self._io_loop = io_loop
self._readers = {} # map of reader objects to fd
self._writers = {} # map of writer objects to fd
@@ -177,8 +186,12 @@ class TornadoReactor(PosixReactorBase):
def callFromThread(self, f, *args, **kw):
"""See `twisted.internet.interfaces.IReactorThreads.callFromThread`"""
assert callable(f), "%s is not callable" % f
p = functools.partial(f, *args, **kw)
self._io_loop.add_callback(p)
with NullContext():
# This NullContext is mainly for an edge case when running
# TwistedIOLoop on top of a TornadoReactor.
# TwistedIOLoop.add_callback uses reactor.callFromThread and
# should not pick up additional StackContexts along the way.
self._io_loop.add_callback(f, *args, **kw)
# We don't need the waker code from the super class, Tornado uses
# its own waker.
@@ -344,7 +357,7 @@ class _TestReactor(TornadoReactor):
def install(io_loop=None):
"""Install this package as the default Twisted reactor."""
if not io_loop:
io_loop = tornado.ioloop.IOLoop.instance()
io_loop = tornado.ioloop.IOLoop.current()
reactor = TornadoReactor(io_loop)
from twisted.internet.main import installReactor
installReactor(reactor)
@@ -383,18 +396,21 @@ class _FD(object):
class TwistedIOLoop(tornado.ioloop.IOLoop):
"""IOLoop implementation that runs on Twisted.
Uses the global Twisted reactor. It is possible to create multiple
TwistedIOLoops in the same process, but it doesn't really make sense
because they will all run in the same thread.
Uses the global Twisted reactor by default. To create multiple
`TwistedIOLoops` in the same process, you must pass a unique reactor
when constructing each one.
Not compatible with `tornado.process.Subprocess.set_exit_callback`
because the ``SIGCHLD`` handlers used by Tornado and Twisted conflict
with each other.
"""
def initialize(self):
from twisted.internet import reactor
def initialize(self, reactor=None):
if reactor is None:
import twisted.internet.reactor
reactor = twisted.internet.reactor
self.reactor = reactor
self.fds = {}
self.reactor.callWhenRunning(self.make_current)
def close(self, all_fds=False):
self.reactor.removeAll()
@@ -456,13 +472,14 @@ class TwistedIOLoop(tornado.ioloop.IOLoop):
if isinstance(deadline, (int, long, float)):
delay = max(deadline - self.time(), 0)
elif isinstance(deadline, datetime.timedelta):
delay = deadline.total_seconds()
delay = tornado.ioloop._Timeout.timedelta_to_seconds(deadline)
else:
raise TypeError("Unsupported deadline %r")
return self.reactor.callLater(delay, self._run_callback, wrap(callback))
def remove_timeout(self, timeout):
timeout.cancel()
if timeout.active():
timeout.cancel()
def add_callback(self, callback, *args, **kwargs):
self.reactor.callFromThread(self._run_callback,
@@ -470,3 +487,58 @@ class TwistedIOLoop(tornado.ioloop.IOLoop):
def add_callback_from_signal(self, callback, *args, **kwargs):
self.add_callback(callback, *args, **kwargs)
class TwistedResolver(Resolver):
"""Twisted-based asynchronous resolver.
This is a non-blocking and non-threaded resolver. It is
recommended only when threads cannot be used, since it has
limitations compared to the standard ``getaddrinfo``-based
`~tornado.netutil.Resolver` and
`~tornado.netutil.ThreadedResolver`. Specifically, it returns at
most one result, and arguments other than ``host`` and ``family``
are ignored. It may fail to resolve when ``family`` is not
``socket.AF_UNSPEC``.
Requires Twisted 12.1 or newer.
"""
def initialize(self, io_loop=None):
self.io_loop = io_loop or IOLoop.current()
# partial copy of twisted.names.client.createResolver, which doesn't
# allow for a reactor to be passed in.
self.reactor = tornado.platform.twisted.TornadoReactor(io_loop)
host_resolver = twisted.names.hosts.Resolver('/etc/hosts')
cache_resolver = twisted.names.cache.CacheResolver(reactor=self.reactor)
real_resolver = twisted.names.client.Resolver('/etc/resolv.conf',
reactor=self.reactor)
self.resolver = twisted.names.resolve.ResolverChain(
[host_resolver, cache_resolver, real_resolver])
@gen.coroutine
def resolve(self, host, port, family=0):
# getHostByName doesn't accept IP addresses, so if the input
# looks like an IP address just return it immediately.
if twisted.internet.abstract.isIPAddress(host):
resolved = host
resolved_family = socket.AF_INET
elif twisted.internet.abstract.isIPv6Address(host):
resolved = host
resolved_family = socket.AF_INET6
else:
deferred = self.resolver.getHostByName(utf8(host))
resolved = yield gen.Task(deferred.addCallback)
if twisted.internet.abstract.isIPAddress(resolved):
resolved_family = socket.AF_INET
elif twisted.internet.abstract.isIPv6Address(resolved):
resolved_family = socket.AF_INET6
else:
resolved_family = socket.AF_UNSPEC
if family != socket.AF_UNSPEC and family != resolved_family:
raise Exception('Requested socket family %d but got %d' %
(family, resolved_family))
result = [
(resolved_family, (resolved, port)),
]
raise gen.Return(result)
libs/tornado/process.py
+9 -7
View File
@@ -14,7 +14,9 @@
# License for the specific language governing permissions and limitations
# under the License.
"""Utilities for working with multiple processes."""
"""Utilities for working with multiple processes, including both forking
the server into multiple processes and managing subprocesses.
"""
from __future__ import absolute_import, division, print_function, with_statement
@@ -169,8 +171,8 @@ class Subprocess(object):
additions:
* ``stdin``, ``stdout``, and ``stderr`` may have the value
`tornado.process.Subprocess.STREAM`, which will make the corresponding
attribute of the resulting Subprocess a `PipeIOStream`.
``tornado.process.Subprocess.STREAM``, which will make the corresponding
attribute of the resulting Subprocess a `.PipeIOStream`.
* A new keyword argument ``io_loop`` may be used to pass in an IOLoop.
"""
STREAM = object()
@@ -229,15 +231,15 @@ class Subprocess(object):
def initialize(cls, io_loop=None):
"""Initializes the ``SIGCHILD`` handler.
The signal handler is run on an IOLoop to avoid locking issues.
Note that the IOLoop used for signal handling need not be the
The signal handler is run on an `.IOLoop` to avoid locking issues.
Note that the `.IOLoop` used for signal handling need not be the
same one used by individual Subprocess objects (as long as the
IOLoops are each running in separate threads).
``IOLoops`` are each running in separate threads).
"""
if cls._initialized:
return
if io_loop is None:
io_loop = ioloop.IOLoop.instance()
io_loop = ioloop.IOLoop.current()
cls._old_sigchld = signal.signal(
signal.SIGCHLD,
lambda sig, frame: io_loop.add_callback_from_signal(cls._cleanup))
libs/tornado/simple_httpclient.py
+37 -28
View File
@@ -2,7 +2,7 @@
from __future__ import absolute_import, division, print_function, with_statement
from tornado.escape import utf8, _unicode, native_str
from tornado.httpclient import HTTPRequest, HTTPResponse, HTTPError, AsyncHTTPClient, main, _RequestProxy
from tornado.httpclient import HTTPResponse, HTTPError, AsyncHTTPClient, main, _RequestProxy
from tornado.httputil import HTTPHeaders
from tornado.iostream import IOStream, SSLIOStream
from tornado.netutil import Resolver, OverrideResolver
@@ -92,10 +92,12 @@ class SimpleAsyncHTTPClient(AsyncHTTPClient):
request, callback = self.queue.popleft()
key = object()
self.active[key] = (request, callback)
_HTTPConnection(self.io_loop, self, request,
functools.partial(self._release_fetch, key),
callback,
self.max_buffer_size, self.resolver)
release_callback = functools.partial(self._release_fetch, key)
self._handle_request(request, release_callback, callback)
def _handle_request(self, request, release_callback, final_callback):
_HTTPConnection(self.io_loop, self, request, release_callback,
final_callback, self.max_buffer_size, self.resolver)
def _release_fetch(self, key):
del self.active[key]
@@ -150,13 +152,24 @@ class _HTTPConnection(object):
# so restrict to ipv4 by default.
af = socket.AF_INET
self.resolver.getaddrinfo(
host, port, af, socket.SOCK_STREAM, 0, 0,
callback=self._on_resolve)
self.resolver.resolve(host, port, af, callback=self._on_resolve)
def _on_resolve(self, future):
af, socktype, proto, canonname, sockaddr = future.result()[0]
def _on_resolve(self, addrinfo):
self.stream = self._create_stream(addrinfo)
timeout = min(self.request.connect_timeout, self.request.request_timeout)
if timeout:
self._timeout = self.io_loop.add_timeout(
self.start_time + timeout,
stack_context.wrap(self._on_timeout))
self.stream.set_close_callback(self._on_close)
# ipv6 addresses are broken (in self.parsed.hostname) until
# 2.7, here is correctly parsed value calculated in __init__
sockaddr = addrinfo[0][1]
self.stream.connect(sockaddr, self._on_connect,
server_hostname=self.parsed_hostname)
def _create_stream(self, addrinfo):
af = addrinfo[0][0]
if self.parsed.scheme == "https":
ssl_options = {}
if self.request.validate_cert:
@@ -189,24 +202,14 @@ class _HTTPConnection(object):
# information.
ssl_options["ssl_version"] = ssl.PROTOCOL_SSLv3
self.stream = SSLIOStream(socket.socket(af, socktype, proto),
io_loop=self.io_loop,
ssl_options=ssl_options,
max_buffer_size=self.max_buffer_size)
return SSLIOStream(socket.socket(af),
io_loop=self.io_loop,
ssl_options=ssl_options,
max_buffer_size=self.max_buffer_size)
else:
self.stream = IOStream(socket.socket(af, socktype, proto),
io_loop=self.io_loop,
max_buffer_size=self.max_buffer_size)
timeout = min(self.request.connect_timeout, self.request.request_timeout)
if timeout:
self._timeout = self.io_loop.add_timeout(
self.start_time + timeout,
stack_context.wrap(self._on_timeout))
self.stream.set_close_callback(self._on_close)
# ipv6 addresses are broken (in self.parsed.hostname) until
# 2.7, here is correctly parsed value calculated in __init__
self.stream.connect(sockaddr, self._on_connect,
server_hostname=self.parsed_hostname)
return IOStream(socket.socket(af),
io_loop=self.io_loop,
max_buffer_size=self.max_buffer_size)
def _on_timeout(self):
self._timeout = None
@@ -246,6 +249,9 @@ class _HTTPConnection(object):
username = self.request.auth_username
password = self.request.auth_password or ''
if username is not None:
if self.request.auth_mode not in (None, "basic"):
raise ValueError("unsupported auth_mode %s",
self.request.auth_mode)
auth = utf8(username) + b":" + utf8(password)
self.request.headers["Authorization"] = (b"Basic " +
base64.b64encode(auth))
@@ -414,7 +420,7 @@ class _HTTPConnection(object):
self.final_callback = None
self._release()
self.client.fetch(new_request, final_callback)
self.stream.close()
self._on_end_request()
return
if self._decompressor:
data = (self._decompressor.decompress(data) +
@@ -434,6 +440,9 @@ class _HTTPConnection(object):
buffer=buffer,
effective_url=self.request.url)
self._run_callback(response)
self._on_end_request()
def _on_end_request(self):
self.stream.close()
def _on_chunk_length(self, data):
libs/tornado/stack_context.py
+121 -99
View File
@@ -14,20 +14,21 @@
# License for the specific language governing permissions and limitations
# under the License.
"""StackContext allows applications to maintain threadlocal-like state
"""`StackContext` allows applications to maintain threadlocal-like state
that follows execution as it moves to other execution contexts.
The motivating examples are to eliminate the need for explicit
async_callback wrappers (as in tornado.web.RequestHandler), and to
``async_callback`` wrappers (as in `tornado.web.RequestHandler`), and to
allow some additional context to be kept for logging.
This is slightly magic, but it's an extension of the idea that an exception
handler is a kind of stack-local state and when that stack is suspended
and resumed in a new context that state needs to be preserved. StackContext
shifts the burden of restoring that state from each call site (e.g.
wrapping each AsyncHTTPClient callback in async_callback) to the mechanisms
that transfer control from one context to another (e.g. AsyncHTTPClient
itself, IOLoop, thread pools, etc).
This is slightly magic, but it's an extension of the idea that an
exception handler is a kind of stack-local state and when that stack
is suspended and resumed in a new context that state needs to be
preserved. `StackContext` shifts the burden of restoring that state
from each call site (e.g. wrapping each `.AsyncHTTPClient` callback
in ``async_callback``) to the mechanisms that transfer control from
one context to another (e.g. `.AsyncHTTPClient` itself, `.IOLoop`,
thread pools, etc).
Example usage::
@@ -52,7 +53,7 @@ Here are a few rules of thumb for when it's necessary:
* If you're writing an asynchronous library that doesn't rely on a
stack_context-aware library like `tornado.ioloop` or `tornado.iostream`
(for example, if you're writing a thread pool), use
`stack_context.wrap()` before any asynchronous operations to capture the
`.stack_context.wrap()` before any asynchronous operations to capture the
stack context from where the operation was started.
* If you're writing an asynchronous library that has some shared
@@ -68,9 +69,6 @@ Here are a few rules of thumb for when it's necessary:
from __future__ import absolute_import, division, print_function, with_statement
import contextlib
import functools
import operator
import sys
import threading
@@ -83,7 +81,7 @@ class StackContextInconsistentError(Exception):
class _State(threading.local):
def __init__(self):
self.contexts = ()
self.contexts = (tuple(), None)
_state = _State()
@@ -107,34 +105,41 @@ class StackContext(object):
context that are currently pending). This is an advanced feature
and not necessary in most applications.
"""
def __init__(self, context_factory, _active_cell=None):
def __init__(self, context_factory):
self.context_factory = context_factory
self.active_cell = _active_cell or [True]
self.contexts = []
# StackContext protocol
def enter(self):
context = self.context_factory()
self.contexts.append(context)
context.__enter__()
def exit(self, type, value, traceback):
context = self.contexts.pop()
context.__exit__(type, value, traceback)
# Note that some of this code is duplicated in ExceptionStackContext
# below. ExceptionStackContext is more common and doesn't need
# the full generality of this class.
def __enter__(self):
self.old_contexts = _state.contexts
# _state.contexts is a tuple of (class, arg, active_cell) tuples
self.new_contexts = (self.old_contexts +
((StackContext, self.context_factory,
self.active_cell),))
self.new_contexts = (self.old_contexts[0] + (self,), self)
_state.contexts = self.new_contexts
try:
self.context = self.context_factory()
self.context.__enter__()
except Exception:
self.enter()
except:
_state.contexts = self.old_contexts
raise
return lambda: operator.setitem(self.active_cell, 0, False)
def __exit__(self, type, value, traceback):
try:
return self.context.__exit__(type, value, traceback)
self.exit(type, value, traceback)
finally:
final_contexts = _state.contexts
_state.contexts = self.old_contexts
# Generator coroutines and with-statements with non-local
# effects interact badly. Check here for signs of
# the stack getting out of sync.
@@ -145,33 +150,32 @@ class StackContext(object):
raise StackContextInconsistentError(
'stack_context inconsistency (may be caused by yield '
'within a "with StackContext" block)')
self.old_contexts = self.new_contexts = None
class ExceptionStackContext(object):
"""Specialization of StackContext for exception handling.
The supplied exception_handler function will be called in the
The supplied ``exception_handler`` function will be called in the
event of an uncaught exception in this context. The semantics are
similar to a try/finally clause, and intended use cases are to log
an error, close a socket, or similar cleanup actions. The
exc_info triple (type, value, traceback) will be passed to the
``exc_info`` triple ``(type, value, traceback)`` will be passed to the
exception_handler function.
If the exception handler returns true, the exception will be
consumed and will not be propagated to other exception handlers.
"""
def __init__(self, exception_handler, _active_cell=None):
def __init__(self, exception_handler):
self.exception_handler = exception_handler
self.active_cell = _active_cell or [True]
def exit(self, type, value, traceback):
if type is not None:
return self.exception_handler(type, value, traceback)
def __enter__(self):
self.old_contexts = _state.contexts
self.new_contexts = (self.old_contexts +
((ExceptionStackContext, self.exception_handler,
self.active_cell),))
self.new_contexts = (self.old_contexts[0], self)
_state.contexts = self.new_contexts
return lambda: operator.setitem(self.active_cell, 0, False)
def __exit__(self, type, value, traceback):
try:
@@ -180,98 +184,116 @@ class ExceptionStackContext(object):
finally:
final_contexts = _state.contexts
_state.contexts = self.old_contexts
if final_contexts is not self.new_contexts:
raise StackContextInconsistentError(
'stack_context inconsistency (may be caused by yield '
'within a "with StackContext" block)')
self.old_contexts = self.new_contexts = None
class NullContext(object):
"""Resets the StackContext.
"""Resets the `StackContext`.
Useful when creating a shared resource on demand (e.g. an AsyncHTTPClient)
where the stack that caused the creating is not relevant to future
operations.
Useful when creating a shared resource on demand (e.g. an
`.AsyncHTTPClient`) where the stack that caused the creating is
not relevant to future operations.
"""
def __enter__(self):
self.old_contexts = _state.contexts
_state.contexts = ()
_state.contexts = (tuple(), None)
def __exit__(self, type, value, traceback):
_state.contexts = self.old_contexts
class _StackContextWrapper(functools.partial):
pass
def wrap(fn):
"""Returns a callable object that will restore the current StackContext
"""Returns a callable object that will restore the current `StackContext`
when executed.
Use this whenever saving a callback to be executed later in a
different execution context (either in a different thread or
asynchronously in the same thread).
"""
if fn is None or fn.__class__ is _StackContextWrapper:
# Check if function is already wrapped
if fn is None or hasattr(fn, '_wrapped'):
return fn
# functools.wraps doesn't appear to work on functools.partial objects
#@functools.wraps(fn)
# Capture current stack head
contexts = _state.contexts
#@functools.wraps
def wrapped(*args, **kwargs):
callback, contexts, args = args[0], args[1], args[2:]
try:
# Force local state - switch to new stack chain
current_state = _state.contexts
_state.contexts = contexts
if _state.contexts:
new_contexts = [NullContext()]
else:
new_contexts = []
if contexts:
new_contexts.extend(cls(arg, active_cell)
for (cls, arg, active_cell) in contexts
if active_cell[0])
if len(new_contexts) > 1:
with _nested(*new_contexts):
callback(*args, **kwargs)
elif new_contexts:
with new_contexts[0]:
callback(*args, **kwargs)
else:
callback(*args, **kwargs)
return _StackContextWrapper(wrapped, fn, _state.contexts)
# Current exception
exc = (None, None, None)
top = None
# Apply stack contexts
last_ctx = 0
stack = contexts[0]
# Apply state
for n in stack:
try:
n.enter()
last_ctx += 1
except:
# Exception happened. Record exception info and store top-most handler
exc = sys.exc_info()
top = n.old_contexts[1]
# Execute callback if no exception happened while restoring state
if top is None:
try:
fn(*args, **kwargs)
except:
exc = sys.exc_info()
top = contexts[1]
# If there was exception, try to handle it by going through the exception chain
if top is not None:
exc = _handle_exception(top, exc)
else:
# Otherwise take shorter path and run stack contexts in reverse order
while last_ctx > 0:
last_ctx -= 1
c = stack[last_ctx]
try:
c.exit(*exc)
except:
exc = sys.exc_info()
top = c.old_contexts[1]
break
else:
top = None
# If if exception happened while unrolling, take longer exception handler path
if top is not None:
exc = _handle_exception(top, exc)
# If exception was not handled, raise it
if exc != (None, None, None):
raise_exc_info(exc)
finally:
_state.contexts = current_state
wrapped._wrapped = True
return wrapped
@contextlib.contextmanager
def _nested(*managers):
"""Support multiple context managers in a single with-statement.
def _handle_exception(tail, exc):
while tail is not None:
try:
if tail.exit(*exc):
exc = (None, None, None)
except:
exc = sys.exc_info()
Copied from the python 2.6 standard library. It's no longer present
in python 3 because the with statement natively supports multiple
context managers, but that doesn't help if the list of context
managers is not known until runtime.
"""
exits = []
vars = []
exc = (None, None, None)
try:
for mgr in managers:
exit = mgr.__exit__
enter = mgr.__enter__
vars.append(enter())
exits.append(exit)
yield vars
except:
exc = sys.exc_info()
finally:
while exits:
exit = exits.pop()
try:
if exit(*exc):
exc = (None, None, None)
except:
exc = sys.exc_info()
if exc != (None, None, None):
# Don't rely on sys.exc_info() still containing
# the right information. Another exception may
# have been raised and caught by an exit method
raise_exc_info(exc)
tail = tail.old_contexts[1]
return exc
libs/tornado/tcpserver.py
+12 -12
View File
@@ -28,13 +28,13 @@ from tornado.iostream import IOStream, SSLIOStream
from tornado.netutil import bind_sockets, add_accept_handler, ssl_wrap_socket
from tornado import process
class TCPServer(object):
r"""A non-blocking, single-threaded TCP server.
To use `TCPServer`, define a subclass which overrides the `handle_stream`
method.
`TCPServer` can serve SSL traffic with Python 2.6+ and OpenSSL.
To make this server serve SSL traffic, send the ssl_options dictionary
argument with the arguments required for the `ssl.wrap_socket` method,
including "certfile" and "keyfile"::
@@ -59,9 +59,9 @@ class TCPServer(object):
server.start(0) # Forks multiple sub-processes
IOLoop.instance().start()
When using this interface, an `IOLoop` must *not* be passed
When using this interface, an `.IOLoop` must *not* be passed
to the `TCPServer` constructor. `start` will always start
the server on the default singleton `IOLoop`.
the server on the default singleton `.IOLoop`.
3. `add_sockets`: advanced multi-process::
@@ -76,7 +76,7 @@ class TCPServer(object):
flexibility in when the fork happens. `add_sockets` can
also be used in single-process servers if you want to create
your listening sockets in some way other than
`bind_sockets`.
`~tornado.netutil.bind_sockets`.
"""
def __init__(self, io_loop=None, ssl_options=None):
self.io_loop = io_loop
@@ -108,7 +108,7 @@ class TCPServer(object):
This method may be called more than once to listen on multiple ports.
`listen` takes effect immediately; it is not necessary to call
`TCPServer.start` afterwards. It is, however, necessary to start
the `IOLoop`.
the `.IOLoop`.
"""
sockets = bind_sockets(port, address=address)
self.add_sockets(sockets)
@@ -117,13 +117,13 @@ class TCPServer(object):
"""Makes this server start accepting connections on the given sockets.
The ``sockets`` parameter is a list of socket objects such as
those returned by `bind_sockets`.
those returned by `~tornado.netutil.bind_sockets`.
`add_sockets` is typically used in combination with that
method and `tornado.process.fork_processes` to provide greater
control over the initialization of a multi-process server.
"""
if self.io_loop is None:
self.io_loop = IOLoop.instance()
self.io_loop = IOLoop.current()
for sock in sockets:
self._sockets[sock.fileno()] = sock
@@ -144,12 +144,12 @@ class TCPServer(object):
Address may be either an IP address or hostname. If it's a hostname,
the server will listen on all IP addresses associated with the
name. Address may be an empty string or None to listen on all
available interfaces. Family may be set to either ``socket.AF_INET``
or ``socket.AF_INET6`` to restrict to ipv4 or ipv6 addresses, otherwise
available interfaces. Family may be set to either `socket.AF_INET`
or `socket.AF_INET6` to restrict to IPv4 or IPv6 addresses, otherwise
both will be used if available.
The ``backlog`` argument has the same meaning as for
`socket.listen`.
`socket.listen <socket.socket.listen>`.
This method may be called multiple times prior to `start` to listen
on multiple ports or interfaces.
@@ -162,7 +162,7 @@ class TCPServer(object):
self._pending_sockets.extend(sockets)
def start(self, num_processes=1):
"""Starts this server in the IOLoop.
"""Starts this server in the `.IOLoop`.
By default, we run the server in this process and do not fork any
additional child process.
@@ -199,7 +199,7 @@ class TCPServer(object):
sock.close()
def handle_stream(self, stream, address):
"""Override to handle a new `IOStream` from an incoming connection."""
"""Override to handle a new `.IOStream` from an incoming connection."""
raise NotImplementedError()
def _handle_connection(self, connection, address):
libs/tornado/template.py
+11 -14
View File
@@ -79,9 +79,10 @@ We provide the functions escape(), url_escape(), json_encode(), and squeeze()
to all templates by default.
Typical applications do not create `Template` or `Loader` instances by
hand, but instead use the `render` and `render_string` methods of
hand, but instead use the `~.RequestHandler.render` and
`~.RequestHandler.render_string` methods of
`tornado.web.RequestHandler`, which load templates automatically based
on the ``template_path`` `Application` setting.
on the ``template_path`` `.Application` setting.
Syntax Reference
----------------
@@ -109,7 +110,7 @@ with ``{# ... #}``.
``{% autoescape *function* %}``
Sets the autoescape mode for the current file. This does not affect
other files, even those referenced by ``{% include %}``. Note that
autoescaping can also be configured globally, at the `Application`
autoescaping can also be configured globally, at the `.Application`
or `Loader`.::
{% autoescape xhtml_escape %}
@@ -298,14 +299,14 @@ class Template(object):
class BaseLoader(object):
"""Base class for template loaders."""
"""Base class for template loaders.
You must use a template loader to use template constructs like
``{% extends %}`` and ``{% include %}``. The loader caches all
templates after they are loaded the first time.
"""
def __init__(self, autoescape=_DEFAULT_AUTOESCAPE, namespace=None):
"""Creates a template loader.
root_directory may be the empty string if this loader does not
use the filesystem.
autoescape must be either None or a string naming a function
"""``autoescape`` must be either None or a string naming a function
in the template namespace, such as "xhtml_escape".
"""
self.autoescape = autoescape
@@ -341,10 +342,6 @@ class BaseLoader(object):
class Loader(BaseLoader):
"""A template loader that loads from a single root directory.
You must use a template loader to use template constructs like
{% extends %} and {% include %}. Loader caches all templates after
they are loaded the first time.
"""
def __init__(self, root_directory, **kwargs):
super(Loader, self).__init__(**kwargs)
libs/tornado/testing.py
+130 -92
View File
@@ -1,21 +1,13 @@
#!/usr/bin/env python
"""Support classes for automated testing.
This module contains three parts:
* `AsyncTestCase` and `AsyncHTTPTestCase`: Subclasses of unittest.TestCase
with additional support for testing asynchronous (`.IOLoop` based) code.
* `AsyncTestCase`/`AsyncHTTPTestCase`: Subclasses of unittest.TestCase
with additional support for testing asynchronous (IOLoop-based) code.
* `LogTrapTestCase`: Subclass of unittest.TestCase that discards log output
from tests that pass and only produces output for failing tests.
* `ExpectLog` and `LogTrapTestCase`: Make test logs less spammy.
* `main()`: A simple test runner (wrapper around unittest.main()) with support
for the tornado.autoreload module to rerun the tests when code changes.
These components may be used together or independently. In particular,
it is safe to combine AsyncTestCase and LogTrapTestCase via multiple
inheritance. See the docstrings for each class/function below for more
information.
"""
from __future__ import absolute_import, division, print_function, with_statement
@@ -46,12 +38,11 @@ import re
import signal
import socket
import sys
import types
try:
from io import StringIO # py3
except ImportError:
from cStringIO import StringIO # py2
except ImportError:
from io import StringIO # py3
# Tornado's own test suite requires the updated unittest module
# (either py27+ or unittest2) so tornado.test.util enforces
@@ -91,30 +82,53 @@ def bind_unused_port():
return sock, port
def get_async_test_timeout():
"""Get the global timeout setting for async tests.
Returns a float, the timeout in seconds.
"""
try:
return float(os.environ.get('ASYNC_TEST_TIMEOUT'))
except (ValueError, TypeError):
return 5
class AsyncTestCase(unittest.TestCase):
"""TestCase subclass for testing IOLoop-based asynchronous code.
"""`~unittest.TestCase` subclass for testing `.IOLoop`-based
asynchronous code.
The unittest framework is synchronous, so the test must be complete
by the time the test method returns. This method provides the stop()
and wait() methods for this purpose. The test method itself must call
self.wait(), and asynchronous callbacks should call self.stop() to signal
completion.
The unittest framework is synchronous, so the test must be
complete by the time the test method returns. This class provides
the `stop()` and `wait()` methods for this purpose. The test
method itself must call ``self.wait()``, and asynchronous
callbacks should call ``self.stop()`` to signal completion.
Alternately, the `gen_test` decorator can be used to use yield points
from the `tornado.gen` module.
By default, a new IOLoop is constructed for each test and is available
as self.io_loop. This IOLoop should be used in the construction of
By default, a new `.IOLoop` is constructed for each test and is available
as ``self.io_loop``. This `.IOLoop` should be used in the construction of
HTTP clients/servers, etc. If the code being tested requires a
global IOLoop, subclasses should override get_new_ioloop to return it.
global `.IOLoop`, subclasses should override `get_new_ioloop` to return it.
The IOLoop's start and stop methods should not be called directly.
Instead, use self.stop self.wait. Arguments passed to self.stop are
returned from self.wait. It is possible to have multiple
wait/stop cycles in the same test.
The `.IOLoop`'s ``start`` and ``stop`` methods should not be
called directly. Instead, use `self.stop <stop>` and `self.wait
<wait>`. Arguments passed to ``self.stop`` are returned from
``self.wait``. It is possible to have multiple ``wait``/``stop``
cycles in the same test.
Example::
# This test uses an asynchronous style similar to most async
# application code.
# This test uses argument passing between self.stop and self.wait.
class MyTestCase(AsyncTestCase):
def test_http_fetch(self):
client = AsyncHTTPClient(self.io_loop)
client.fetch("http://www.tornadoweb.org/", self.stop)
response = self.wait()
# Test contents of response
self.assertIn("FriendFeed", response.body)
# This test uses an explicit callback-based style.
class MyTestCase2(AsyncTestCase):
def test_http_fetch(self):
client = AsyncHTTPClient(self.io_loop)
client.fetch("http://www.tornadoweb.org/", self.handle_fetch)
@@ -128,19 +142,6 @@ class AsyncTestCase(unittest.TestCase):
# self.wait() in test_http_fetch() via stack_context.
self.assertIn("FriendFeed", response.body)
self.stop()
# This test uses the argument passing between self.stop and self.wait
# for a simpler, more synchronous style.
# This style is recommended over the preceding example because it
# keeps the assertions in the test method itself, and is therefore
# less sensitive to the subtleties of stack_context.
class MyTestCase2(AsyncTestCase):
def test_http_fetch(self):
client = AsyncHTTPClient(self.io_loop)
client.fetch("http://www.tornadoweb.org/", self.stop)
response = self.wait()
# Test contents of response
self.assertIn("FriendFeed", response.body)
"""
def __init__(self, *args, **kwargs):
super(AsyncTestCase, self).__init__(*args, **kwargs)
@@ -165,16 +166,21 @@ class AsyncTestCase(unittest.TestCase):
# set FD_CLOEXEC on its file descriptors)
self.io_loop.close(all_fds=True)
super(AsyncTestCase, self).tearDown()
# In case an exception escaped or the StackContext caught an exception
# when there wasn't a wait() to re-raise it, do so here.
# This is our last chance to raise an exception in a way that the
# unittest machinery understands.
self.__rethrow()
def get_new_ioloop(self):
"""Creates a new IOLoop for this test. May be overridden in
subclasses for tests that require a specific IOLoop (usually
the singleton).
"""Creates a new `.IOLoop` for this test. May be overridden in
subclasses for tests that require a specific `.IOLoop` (usually
the singleton `.IOLoop.instance()`).
"""
return IOLoop()
def _handle_exception(self, typ, value, tb):
self.__failure = sys.exc_info()
self.__failure = (typ, value, tb)
self.stop()
return True
@@ -187,16 +193,18 @@ class AsyncTestCase(unittest.TestCase):
def run(self, result=None):
with ExceptionStackContext(self._handle_exception):
super(AsyncTestCase, self).run(result)
# In case an exception escaped super.run or the StackContext caught
# an exception when there wasn't a wait() to re-raise it, do so here.
# As a last resort, if an exception escaped super.run() and wasn't
# re-raised in tearDown, raise it here. This will cause the
# unittest run to fail messily, but that's better than silently
# ignoring an error.
self.__rethrow()
def stop(self, _arg=None, **kwargs):
"""Stops the ioloop, causing one pending (or future) call to wait()
"""Stops the `.IOLoop`, causing one pending (or future) call to `wait()`
to return.
Keyword arguments or a single positional argument passed to stop() are
saved and will be returned by wait().
Keyword arguments or a single positional argument passed to `stop()` are
saved and will be returned by `wait()`.
"""
assert _arg is None or not kwargs
self.__stop_args = kwargs or _arg
@@ -205,14 +213,19 @@ class AsyncTestCase(unittest.TestCase):
self.__running = False
self.__stopped = True
def wait(self, condition=None, timeout=5):
"""Runs the IOLoop until stop is called or timeout has passed.
def wait(self, condition=None, timeout=None):
"""Runs the `.IOLoop` until stop is called or timeout has passed.
In the event of a timeout, an exception will be thrown.
In the event of a timeout, an exception will be thrown. The default
timeout is 5 seconds; it may be overridden with a ``timeout`` keyword
argument or globally with the ASYNC_TEST_TIMEOUT environment variable.
If condition is not None, the IOLoop will be restarted after stop()
until condition() returns true.
If ``condition`` is not None, the `.IOLoop` will be restarted
after `stop()` until ``condition()`` returns true.
"""
if timeout is None:
timeout = get_async_test_timeout()
if not self.__stopped:
if timeout:
def timeout_func():
@@ -244,9 +257,9 @@ class AsyncTestCase(unittest.TestCase):
class AsyncHTTPTestCase(AsyncTestCase):
"""A test case that starts up an HTTP server.
Subclasses must override get_app(), which returns the
tornado.web.Application (or other HTTPServer callback) to be tested.
Tests will typically use the provided self.http_client to fetch
Subclasses must override `get_app()`, which returns the
`tornado.web.Application` (or other `.HTTPServer` callback) to be tested.
Tests will typically use the provided ``self.http_client`` to fetch
URLs from this server.
Example::
@@ -283,17 +296,17 @@ class AsyncHTTPTestCase(AsyncTestCase):
def get_app(self):
"""Should be overridden by subclasses to return a
tornado.web.Application or other HTTPServer callback.
`tornado.web.Application` or other `.HTTPServer` callback.
"""
raise NotImplementedError()
def fetch(self, path, **kwargs):
"""Convenience method to synchronously fetch a url.
The given path will be appended to the local server's host and port.
Any additional kwargs will be passed directly to
AsyncHTTPClient.fetch (and so could be used to pass method="POST",
body="...", etc).
The given path will be appended to the local server's host and
port. Any additional kwargs will be passed directly to
`.AsyncHTTPClient.fetch` (and so could be used to pass
``method="POST"``, ``body="..."``, etc).
"""
self.http_client.fetch(self.get_url(path), self.stop, **kwargs)
return self.wait()
@@ -357,34 +370,58 @@ class AsyncHTTPSTestCase(AsyncHTTPTestCase):
return 'https'
def gen_test(f):
"""Testing equivalent of ``@gen.engine``, to be applied to test methods.
def gen_test(func=None, timeout=None):
"""Testing equivalent of ``@gen.coroutine``, to be applied to test methods.
``@gen.engine`` cannot be used on tests because the `IOLoop` is not
``@gen.coroutine`` cannot be used on tests because the `.IOLoop` is not
already running. ``@gen_test`` should be applied to test methods
on subclasses of `AsyncTestCase`.
Note that unlike most uses of ``@gen.engine``, ``@gen_test`` can
detect automatically when the function finishes cleanly so there
is no need to run a callback to signal completion.
Example::
class MyTest(AsyncHTTPTestCase):
@gen_test
def test_something(self):
response = yield gen.Task(self.fetch('/'))
By default, ``@gen_test`` times out after 5 seconds. The timeout may be
overridden globally with the ASYNC_TEST_TIMEOUT environment variable,
or for each test with the ``timeout`` keyword argument::
class MyTest(AsyncHTTPTestCase):
@gen_test(timeout=10)
def test_something_slow(self):
response = yield gen.Task(self.fetch('/'))
If both the environment variable and the parameter are set, ``gen_test``
uses the maximum of the two.
"""
@functools.wraps(f)
def wrapper(self, *args, **kwargs):
result = f(self, *args, **kwargs)
if result is None:
return
assert isinstance(result, types.GeneratorType)
runner = gen.Runner(result, self.stop)
runner.run()
self.wait()
return wrapper
if timeout is None:
timeout = get_async_test_timeout()
def wrap(f):
f = gen.coroutine(f)
@functools.wraps(f)
def wrapper(self):
return self.io_loop.run_sync(
functools.partial(f, self), timeout=timeout)
return wrapper
if func is not None:
# Used like:
# @gen_test
# def f(self):
# pass
return wrap(func)
else:
# Used like @gen_test(timeout=10)
return wrap
# Without this attribute, nosetests will try to run gen_test as a test
# anywhere it is imported.
gen_test.__test__ = False
class LogTrapTestCase(unittest.TestCase):
@@ -395,13 +432,13 @@ class LogTrapTestCase(unittest.TestCase):
the test succeeds, so this class can be useful to minimize the noise.
Simply use it as a base class for your test case. It is safe to combine
with AsyncTestCase via multiple inheritance
("class MyTestCase(AsyncHTTPTestCase, LogTrapTestCase):")
(``class MyTestCase(AsyncHTTPTestCase, LogTrapTestCase):``)
This class assumes that only one log handler is configured and that
it is a StreamHandler. This is true for both logging.basicConfig
and the "pretty logging" configured by tornado.options. It is not
compatible with other log buffering mechanisms, such as those provided
by some test runners.
This class assumes that only one log handler is configured and
that it is a `~logging.StreamHandler`. This is true for both
`logging.basicConfig` and the "pretty logging" configured by
`tornado.options`. It is not compatible with other log buffering
mechanisms, such as those provided by some test runners.
"""
def run(self, result=None):
logger = logging.getLogger()
@@ -409,7 +446,7 @@ class LogTrapTestCase(unittest.TestCase):
logging.basicConfig()
handler = logger.handlers[0]
if (len(logger.handlers) > 1 or
not isinstance(handler, logging.StreamHandler)):
not isinstance(handler, logging.StreamHandler)):
# Logging has been configured in a way we don't recognize,
# so just leave it alone.
super(LogTrapTestCase, self).run(result)
@@ -486,10 +523,11 @@ def main(**kwargs):
be specified.
Projects with many tests may wish to define a test script like
tornado/test/runtests.py. This script should define a method all()
which returns a test suite and then call tornado.testing.main().
Note that even when a test script is used, the all() test suite may
be overridden by naming a single test on the command line::
``tornado/test/runtests.py``. This script should define a method
``all()`` which returns a test suite and then call
`tornado.testing.main()`. Note that even when a test script is
used, the ``all()`` test suite may be overridden by naming a
single test on the command line::
# Runs all tests
python -m tornado.test.runtests
libs/tornado/util.py
+37 -13
View File
@@ -1,4 +1,14 @@
"""Miscellaneous utility functions."""
"""Miscellaneous utility functions and classes.
This module is used internally by Tornado. It is not necessarily expected
that the functions and classes defined here will be useful to other
applications, but they are documented here in case they are.
The one public-facing part of this module is the `Configurable` class
and its `~Configurable.configure` method, which becomes a part of the
interface of its subclasses, including `.AsyncHTTPClient`, `.IOLoop`,
and `.Resolver`.
"""
from __future__ import absolute_import, division, print_function, with_statement
@@ -8,7 +18,8 @@ import zlib
class ObjectDict(dict):
"""Makes a dictionary behave like an object."""
"""Makes a dictionary behave like an object, with attribute-style access.
"""
def __getattr__(self, name):
try:
return self[name]
@@ -52,6 +63,7 @@ class GzipDecompressor(object):
def import_object(name):
"""Imports an object by name.
import_object('x') is equivalent to 'import x'.
import_object('x.y.z') is equivalent to 'from x.y import z'.
>>> import tornado.escape
@@ -59,10 +71,23 @@ def import_object(name):
True
>>> import_object('tornado.escape.utf8') is tornado.escape.utf8
True
>>> import_object('tornado') is tornado
True
>>> import_object('tornado.missing_module')
Traceback (most recent call last):
...
ImportError: No module named missing_module
"""
if name.count('.') == 0:
return __import__(name, None, None)
parts = name.split('.')
obj = __import__('.'.join(parts[:-1]), None, None, [parts[-1]], 0)
return getattr(obj, parts[-1])
try:
return getattr(obj, parts[-1])
except AttributeError:
raise ImportError("No module named %s" % parts[-1])
# Fake unicode literal support: Python 3.2 doesn't have the u'' marker for
# literal strings, and alternative solutions like "from __future__ import
@@ -115,16 +140,17 @@ class Configurable(object):
The implementation subclass as well as optional keyword arguments to
its initializer can be set globally at runtime with `configure`.
By using the constructor as the factory method, the interface looks like
a normal class, ``isinstance()`` works as usual, etc. This pattern
is most useful when the choice of implementation is likely to be a
global decision (e.g. when epoll is available, always use it instead of
select), or when a previously-monolithic class has been split into
specialized subclasses.
By using the constructor as the factory method, the interface
looks like a normal class, `isinstance` works as usual, etc. This
pattern is most useful when the choice of implementation is likely
to be a global decision (e.g. when `~select.epoll` is available,
always use it instead of `~select.select`), or when a
previously-monolithic class has been split into specialized
subclasses.
Configurable subclasses must define the class methods
`configurable_base` and `configurable_default`, and use the instance
method `initialize` instead of `__init__`.
method `initialize` instead of ``__init__``.
"""
__impl_class = None
__impl_kwargs = None
@@ -163,7 +189,7 @@ class Configurable(object):
def initialize(self):
"""Initialize a `Configurable` subclass instance.
Configurable classes should use `initialize` instead of `__init__`.
Configurable classes should use `initialize` instead of ``__init__``.
"""
@classmethod
@@ -210,8 +236,6 @@ class ArgReplacer(object):
and similar wrappers.
"""
def __init__(self, func, name):
"""Create an ArgReplacer for the named argument to the given function.
"""
self.name = name
try:
self.arg_pos = inspect.getargspec(func).args.index(self.name)
libs/tornado/web.py
+162 -118
View File
@@ -14,13 +14,12 @@
# License for the specific language governing permissions and limitations
# under the License.
"""
The Tornado web framework looks a bit like web.py (http://webpy.org/) or
Google's webapp (http://code.google.com/appengine/docs/python/tools/webapp/),
but with additional tools and optimizations to take advantage of the
Tornado non-blocking web server and tools.
"""``tornado.web`` provides a simple web framework with asynchronous
features that allow it to scale to large numbers of open connections,
making it ideal for `long polling
<http://en.wikipedia.org/wiki/Push_technology#Long_polling>`_.
Here is the canonical "Hello, world" example app::
Here is a simple "Hello, world" example app::
import tornado.ioloop
import tornado.web
@@ -36,17 +35,19 @@ Here is the canonical "Hello, world" example app::
application.listen(8888)
tornado.ioloop.IOLoop.instance().start()
See the Tornado walkthrough on http://tornadoweb.org for more details
and a good getting started guide.
See the :doc:`Tornado overview <overview>` for more details and a good getting
started guide.
Thread-safety notes
-------------------
In general, methods on RequestHandler and elsewhere in tornado are not
thread-safe. In particular, methods such as write(), finish(), and
flush() must only be called from the main thread. If you use multiple
threads it is important to use IOLoop.add_callback to transfer control
back to the main thread before finishing the request.
In general, methods on `RequestHandler` and elsewhere in Tornado are
not thread-safe. In particular, methods such as
`~RequestHandler.write()`, `~RequestHandler.finish()`, and
`~RequestHandler.flush()` must only be called from the main thread. If
you use multiple threads it is important to use `.IOLoop.add_callback`
to transfer control back to the main thread before finishing the
request.
"""
from __future__ import absolute_import, division, print_function, with_statement
@@ -72,6 +73,7 @@ import traceback
import types
import uuid
from tornado.concurrent import Future
from tornado import escape
from tornado import httputil
from tornado import locale
@@ -103,11 +105,11 @@ except ImportError:
class RequestHandler(object):
"""Subclass this class and define get() or post() to make a handler.
"""Subclass this class and define `get()` or `post()` to make a handler.
If you want to support more methods than the standard GET/HEAD/POST, you
should override the class variable SUPPORTED_METHODS in your
RequestHandler class.
should override the class variable ``SUPPORTED_METHODS`` in your
`RequestHandler` subclass.
"""
SUPPORTED_METHODS = ("GET", "HEAD", "POST", "DELETE", "PATCH", "PUT",
"OPTIONS")
@@ -167,7 +169,7 @@ class RequestHandler(object):
@property
def settings(self):
"""An alias for `self.application.settings`."""
"""An alias for `self.application.settings <Application.settings>`."""
return self.application.settings
def head(self, *args, **kwargs):
@@ -192,7 +194,7 @@ class RequestHandler(object):
raise HTTPError(405)
def prepare(self):
"""Called at the beginning of a request before `get`/`post`/etc.
"""Called at the beginning of a request before `get`/`post`/etc.
Override this method to perform common initialization regardless
of the request method.
@@ -228,10 +230,10 @@ class RequestHandler(object):
def clear(self):
"""Resets all headers and content for this response."""
self._headers = httputil.HTTPHeaders({
"Server": "TornadoServer/%s" % tornado.version,
"Content-Type": "text/html; charset=UTF-8",
"Date": httputil.format_timestamp(time.gmtime()),
})
"Server": "TornadoServer/%s" % tornado.version,
"Content-Type": "text/html; charset=UTF-8",
"Date": httputil.format_timestamp(time.gmtime()),
})
self.set_default_headers()
if not self.request.supports_http_1_1():
if self.request.headers.get("Connection") == "Keep-Alive":
@@ -253,10 +255,11 @@ class RequestHandler(object):
def set_status(self, status_code, reason=None):
"""Sets the status code for our response.
:arg int status_code: Response status code. If `reason` is ``None``,
it must be present in `httplib.responses`.
:arg int status_code: Response status code. If ``reason`` is ``None``,
it must be present in `httplib.responses <http.client.responses>`.
:arg string reason: Human-readable reason phrase describing the status
code. If ``None``, it will be filled in from `httplib.responses`.
code. If ``None``, it will be filled in from
`httplib.responses <http.client.responses>`.
"""
self._status_code = status_code
if reason is not None:
@@ -363,8 +366,8 @@ class RequestHandler(object):
By default, this method decodes the argument as utf-8 and returns
a unicode string, but this may be overridden in subclasses.
This method is used as a filter for both get_argument() and for
values extracted from the url and passed to get()/post()/etc.
This method is used as a filter for both `get_argument()` and for
values extracted from the url and passed to `get()`/`post()`/etc.
The name of the argument is provided if known, but may be None
(e.g. for unnamed groups in the url regex).
@@ -373,6 +376,7 @@ class RequestHandler(object):
@property
def cookies(self):
"""An alias for `self.request.cookies <.httpserver.HTTPRequest.cookies>`."""
return self.request.cookies
def get_cookie(self, name, default=None):
@@ -496,8 +500,8 @@ class RequestHandler(object):
To write the output to the network, use the flush() method below.
If the given chunk is a dictionary, we write it as JSON and set
the Content-Type of the response to be application/json.
(if you want to send JSON as a different Content-Type, call
the Content-Type of the response to be ``application/json``.
(if you want to send JSON as a different ``Content-Type``, call
set_header *after* calling write()).
Note that lists are not converted to JSON because of a potential
@@ -604,8 +608,8 @@ class RequestHandler(object):
def render_string(self, template_name, **kwargs):
"""Generate the given template with the given arguments.
We return the generated string. To generate and write a template
as a response, use render() above.
We return the generated byte string (in utf8). To generate and
write a template as a response, use render() above.
"""
# If no template_path is specified, use the path of the calling file
template_path = self.get_template_path()
@@ -743,6 +747,9 @@ class RequestHandler(object):
self._log()
self._finished = True
self.on_finish()
# Break up a reference cycle between this handler and the
# _ui_module closures to allow for faster GC on CPython.
self.ui = None
def send_error(self, status_code=500, **kwargs):
"""Sends the given HTTP error code to the browser.
@@ -823,9 +830,9 @@ class RequestHandler(object):
def locale(self):
"""The local for the current session.
Determined by either get_user_locale, which you can override to
Determined by either `get_user_locale`, which you can override to
set the locale based on, e.g., a user preference stored in a
database, or get_browser_locale, which uses the Accept-Language
database, or `get_browser_locale`, which uses the ``Accept-Language``
header.
"""
if not hasattr(self, "_locale"):
@@ -838,15 +845,15 @@ class RequestHandler(object):
def get_user_locale(self):
"""Override to determine the locale from the authenticated user.
If None is returned, we fall back to get_browser_locale().
If None is returned, we fall back to `get_browser_locale()`.
This method should return a tornado.locale.Locale object,
most likely obtained via a call like tornado.locale.get("en")
This method should return a `tornado.locale.Locale` object,
most likely obtained via a call like ``tornado.locale.get("en")``
"""
return None
def get_browser_locale(self, default="en_US"):
"""Determines the user's locale from Accept-Language header.
"""Determines the user's locale from ``Accept-Language`` header.
See http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.4
"""
@@ -873,9 +880,9 @@ class RequestHandler(object):
def current_user(self):
"""The authenticated user for this request.
Determined by either get_current_user, which you can override to
set the user based on, e.g., a cookie. If that method is not
overridden, this method always returns None.
This is a cached version of `get_current_user`, which you can
override to set the user based on, e.g., a cookie. If that
method is not overridden, this method always returns None.
We lazy-load the current user the first time this method is called
and cache the result after that.
@@ -891,7 +898,7 @@ class RequestHandler(object):
def get_login_url(self):
"""Override to customize the login URL based on the request.
By default, we use the 'login_url' application setting.
By default, we use the ``login_url`` application setting.
"""
self.require_setting("login_url", "@tornado.web.authenticated")
return self.application.settings["login_url"]
@@ -899,7 +906,7 @@ class RequestHandler(object):
def get_template_path(self):
"""Override to customize template path for each handler.
By default, we use the 'template_path' application setting.
By default, we use the ``template_path`` application setting.
Return None to load templates relative to the calling file.
"""
return self.application.settings.get("template_path")
@@ -925,21 +932,21 @@ class RequestHandler(object):
return self._xsrf_token
def check_xsrf_cookie(self):
"""Verifies that the '_xsrf' cookie matches the '_xsrf' argument.
"""Verifies that the ``_xsrf`` cookie matches the ``_xsrf`` argument.
To prevent cross-site request forgery, we set an '_xsrf'
To prevent cross-site request forgery, we set an ``_xsrf``
cookie and include the same value as a non-cookie
field with all POST requests. If the two do not match, we
field with all ``POST`` requests. If the two do not match, we
reject the form submission as a potential forgery.
The _xsrf value may be set as either a form field named _xsrf
or in a custom HTTP header named X-XSRFToken or X-CSRFToken
The ``_xsrf`` value may be set as either a form field named ``_xsrf``
or in a custom HTTP header named ``X-XSRFToken`` or ``X-CSRFToken``
(the latter is accepted for compatibility with Django).
See http://en.wikipedia.org/wiki/Cross-site_request_forgery
Prior to release 1.1.1, this check was ignored if the HTTP header
"X-Requested-With: XMLHTTPRequest" was present. This exception
``X-Requested-With: XMLHTTPRequest`` was present. This exception
has been shown to be insecure and has been removed. For more
information please see
http://www.djangoproject.com/weblog/2011/feb/08/security/
@@ -954,14 +961,17 @@ class RequestHandler(object):
raise HTTPError(403, "XSRF cookie does not match POST argument")
def xsrf_form_html(self):
"""An HTML <input/> element to be included with all POST forms.
"""An HTML ``<input/>`` element to be included with all POST forms.
It defines the _xsrf input value, which we check on all POST
It defines the ``_xsrf`` input value, which we check on all POST
requests to prevent cross-site request forgery. If you have set
the 'xsrf_cookies' application setting, you must include this
the ``xsrf_cookies`` application setting, you must include this
HTML within all of your HTML forms.
See check_xsrf_cookie() above for more information.
In a template, this method should be called with ``{% module
xsrf_form_html() %}``
See `check_xsrf_cookie()` above for more information.
"""
return '<input type="hidden" name="_xsrf" value="' + \
escape.xhtml_escape(self.xsrf_token) + '"/>'
@@ -969,11 +979,11 @@ class RequestHandler(object):
def static_url(self, path, include_host=None):
"""Returns a static URL for the given relative static file path.
This method requires you set the 'static_path' setting in your
This method requires you set the ``static_path`` setting in your
application (which specifies the root directory of your static
files).
We append ?v=<signature> to the returned URL, which makes our
We append ``?v=<signature>`` to the returned URL, which makes our
static file handler set an infinite expiration header on the
returned content. The signature is based on the content of the
file.
@@ -1142,10 +1152,18 @@ class RequestHandler(object):
def asynchronous(method):
"""Wrap request handler methods with this if they are asynchronous.
This decorator should only be applied to the :ref:`HTTP verb
methods <verbs>`; its behavior is undefined for any other method.
This decorator does not *make* a method asynchronous; it tells
the framework that the method *is* asynchronous. For this decorator
to be useful the method must (at least sometimes) do something
asynchronous.
If this decorator is given, the response is not finished when the
method returns. It is up to the request handler to call self.finish()
to finish the HTTP request. Without this decorator, the request is
automatically finished when the get() or post() method returns. ::
method returns. It is up to the request handler to call
`self.finish() <RequestHandler.finish>` to finish the HTTP
request. Without this decorator, the request is automatically
finished when the ``get()`` or ``post()`` method returns. Example::
class MyRequestHandler(web.RequestHandler):
@web.asynchronous
@@ -1158,6 +1176,8 @@ def asynchronous(method):
self.finish()
"""
# Delay the IOLoop import because it's not available on app engine.
from tornado.ioloop import IOLoop
@functools.wraps(method)
def wrapper(self, *args, **kwargs):
if self.application._wsgi:
@@ -1165,14 +1185,27 @@ def asynchronous(method):
self._auto_finish = False
with stack_context.ExceptionStackContext(
self._stack_context_handle_exception):
return method(self, *args, **kwargs)
result = method(self, *args, **kwargs)
if isinstance(result, Future):
# If @asynchronous is used with @gen.coroutine, (but
# not @gen.engine), we can automatically finish the
# request when the future resolves. Additionally,
# the Future will swallow any exceptions so we need
# to throw them back out to the stack context to finish
# the request.
def future_complete(f):
f.result()
if not self._finished:
self.finish()
IOLoop.current().add_future(result, future_complete)
return result
return wrapper
def removeslash(method):
"""Use this decorator to remove trailing slashes from the request path.
For example, a request to ``'/foo/'`` would redirect to ``'/foo'`` with this
For example, a request to ``/foo/`` would redirect to ``/foo`` with this
decorator. Your request handler mapping should use a regular expression
like ``r'/foo/*'`` in conjunction with using the decorator.
"""
@@ -1195,9 +1228,9 @@ def removeslash(method):
def addslash(method):
"""Use this decorator to add a missing trailing slash to the request path.
For example, a request to '/foo' would redirect to '/foo/' with this
For example, a request to ``/foo`` would redirect to ``/foo/`` with this
decorator. Your request handler mapping should use a regular expression
like r'/foo/?' in conjunction with using the decorator.
like ``r'/foo/?'`` in conjunction with using the decorator.
"""
@functools.wraps(method)
def wrapper(self, *args, **kwargs):
@@ -1226,35 +1259,36 @@ class Application(object):
http_server.listen(8080)
ioloop.IOLoop.instance().start()
The constructor for this class takes in a list of URLSpec objects
The constructor for this class takes in a list of `URLSpec` objects
or (regexp, request_class) tuples. When we receive requests, we
iterate over the list in order and instantiate an instance of the
first request class whose regexp matches the request path.
Each tuple can contain an optional third element, which should be a
dictionary if it is present. That dictionary is passed as keyword
arguments to the contructor of the handler. This pattern is used
for the StaticFileHandler below (note that a StaticFileHandler
can be installed automatically with the static_path setting described
below)::
Each tuple can contain an optional third element, which should be
a dictionary if it is present. That dictionary is passed as
keyword arguments to the contructor of the handler. This pattern
is used for the `StaticFileHandler` in this example (note that a
`StaticFileHandler` can be installed automatically with the
static_path setting described below)::
application = web.Application([
(r"/static/(.*)", web.StaticFileHandler, {"path": "/var/www"}),
])
We support virtual hosts with the add_handlers method, which takes in
We support virtual hosts with the `add_handlers` method, which takes in
a host regular expression as the first argument::
application.add_handlers(r"www\.myhost\.com", [
(r"/article/([0-9]+)", ArticleHandler),
])
You can serve static files by sending the static_path setting as a
keyword argument. We will serve those files from the /static/ URI
(this is configurable with the static_url_prefix setting),
and we will serve /favicon.ico and /robots.txt from the same directory.
A custom subclass of StaticFileHandler can be specified with the
static_handler_class setting.
You can serve static files by sending the ``static_path`` setting
as a keyword argument. We will serve those files from the
``/static/`` URI (this is configurable with the
``static_url_prefix`` setting), and we will serve ``/favicon.ico``
and ``/robots.txt`` from the same directory. A custom subclass of
`StaticFileHandler` can be specified with the
``static_handler_class`` setting.
"""
def __init__(self, handlers=None, default_host="", transforms=None,
wsgi=False, **settings):
@@ -1301,15 +1335,16 @@ class Application(object):
def listen(self, port, address="", **kwargs):
"""Starts an HTTP server for this application on the given port.
This is a convenience alias for creating an HTTPServer object
and calling its listen method. Keyword arguments not
supported by HTTPServer.listen are passed to the HTTPServer
constructor. For advanced uses (e.g. preforking), do not use
this method; create an HTTPServer and call its bind/start
methods directly.
This is a convenience alias for creating an `.HTTPServer`
object and calling its listen method. Keyword arguments not
supported by `HTTPServer.listen <.TCPServer.listen>` are passed to the
`.HTTPServer` constructor. For advanced uses
(e.g. multi-process mode), do not use this method; create an
`.HTTPServer` and call its
`.TCPServer.bind`/`.TCPServer.start` methods directly.
Note that after calling this method you still need to call
IOLoop.instance().start() to start the server.
``IOLoop.instance().start()`` to start the server.
"""
# import is here rather than top level because HTTPServer
# is not importable on appengine
@@ -1337,7 +1372,7 @@ class Application(object):
self.handlers.append((re.compile(host_pattern), handlers))
for spec in host_handlers:
if isinstance(spec, type(())):
if isinstance(spec, (tuple, list)):
assert len(spec) in (2, 3)
pattern = spec[0]
handler = spec[1]
@@ -1361,7 +1396,6 @@ class Application(object):
self.named_handlers[spec.name] = spec
def add_transform(self, transform_class):
"""Adds the given OutputTransform to our transform list."""
self.transforms.append(transform_class)
def _get_host_handlers(self, request):
@@ -1456,11 +1490,11 @@ class Application(object):
return handler
def reverse_url(self, name, *args):
"""Returns a URL path for handler named `name`
"""Returns a URL path for handler named ``name``
The handler must be added to the application as a named URLSpec.
The handler must be added to the application as a named `URLSpec`.
Args will be substituted for capturing groups in the URLSpec regex.
Args will be substituted for capturing groups in the `URLSpec` regex.
They will be converted to strings if necessary, encoded as utf8,
and url-escaped.
"""
@@ -1474,7 +1508,7 @@ class Application(object):
By default writes to the python root logger. To change
this behavior either subclass Application and override this method,
or pass a function in the application settings dictionary as
'log_function'.
``log_function``.
"""
if "log_function" in self.settings:
self.settings["log_function"](handler)
@@ -1493,8 +1527,13 @@ class Application(object):
class HTTPError(Exception):
"""An exception that will turn into an HTTP error response.
Raising an `HTTPError` is a convenient alternative to calling
`RequestHandler.send_error` since it automatically ends the
current function.
:arg int status_code: HTTP status code. Must be listed in
`httplib.responses` unless the ``reason`` keyword argument is given.
`httplib.responses <http.client.responses>` unless the ``reason``
keyword argument is given.
:arg string log_message: Message to be written to the log for this error
(will not be shown to the user unless the `Application` is in debug
mode). May contain ``%s``-style placeholders, which will be filled
@@ -1521,7 +1560,7 @@ class HTTPError(Exception):
class ErrorHandler(RequestHandler):
"""Generates an error response with status_code for all requests."""
"""Generates an error response with ``status_code`` for all requests."""
def initialize(self, status_code):
self.set_status(status_code)
@@ -1538,7 +1577,7 @@ class ErrorHandler(RequestHandler):
class RedirectHandler(RequestHandler):
"""Redirects the client to the given URL for all GET requests.
You should provide the keyword argument "url" to the handler, e.g.::
You should provide the keyword argument ``url`` to the handler, e.g.::
application = web.Application([
(r"/oldpath", web.RedirectHandler, {"url": "/newpath"}),
@@ -1555,20 +1594,20 @@ class RedirectHandler(RequestHandler):
class StaticFileHandler(RequestHandler):
"""A simple handler that can serve static content from a directory.
To map a path to this handler for a static data directory /var/www,
To map a path to this handler for a static data directory ``/var/www``,
you would add a line to your application like::
application = web.Application([
(r"/static/(.*)", web.StaticFileHandler, {"path": "/var/www"}),
])
The local root directory of the content should be passed as the "path"
The local root directory of the content should be passed as the ``path``
argument to the handler.
To support aggressive browser caching, if the argument "v" is given
To support aggressive browser caching, if the argument ``v`` is given
with the path, we set an infinite HTTP expiration header. So, if you
want browsers to cache a file indefinitely, send them to, e.g.,
/static/images/myimage.png?v=xxx. Override ``get_cache_time`` method for
``/static/images/myimage.png?v=xxx``. Override `get_cache_time` method for
more fine-grained cache control.
"""
CACHE_MAX_AGE = 86400 * 365 * 10 # 10 years
@@ -1609,7 +1648,7 @@ class StaticFileHandler(RequestHandler):
raise HTTPError(403, "%s is not a file", path)
stat_result = os.stat(abspath)
modified = datetime.datetime.fromtimestamp(stat_result[stat.ST_MTIME])
modified = datetime.datetime.utcfromtimestamp(stat_result[stat.ST_MTIME])
self.set_header("Last-Modified", modified)
@@ -1631,7 +1670,7 @@ class StaticFileHandler(RequestHandler):
ims_value = self.request.headers.get("If-Modified-Since")
if ims_value is not None:
date_tuple = email.utils.parsedate(ims_value)
if_since = datetime.datetime.fromtimestamp(time.mktime(date_tuple))
if_since = datetime.datetime(*date_tuple[:6])
if if_since >= modified:
self.set_status(304)
return
@@ -1651,11 +1690,13 @@ class StaticFileHandler(RequestHandler):
def get_cache_time(self, path, modified, mime_type):
"""Override to customize cache control behavior.
Return a positive number of seconds to trigger aggressive caching or 0
to mark resource as cacheable, only.
Return a positive number of seconds to make the result
cacheable for that amount of time or 0 to mark resource as
cacheable for an unspecified amount of time (subject to
browser heuristics).
By default returns cache expiry of 10 years for resources requested
with "v" argument.
with ``v`` argument.
"""
return self.CACHE_MAX_AGE if "v" in self.request.arguments else 0
@@ -1718,12 +1759,13 @@ class StaticFileHandler(RequestHandler):
class FallbackHandler(RequestHandler):
"""A RequestHandler that wraps another HTTP server callback.
"""A `RequestHandler` that wraps another HTTP server callback.
The fallback is a callable object that accepts an HTTPRequest,
such as an Application or tornado.wsgi.WSGIContainer. This is most
useful to use both tornado RequestHandlers and WSGI in the same server.
Typical usage::
The fallback is a callable object that accepts an
`~.httpserver.HTTPRequest`, such as an `Application` or
`tornado.wsgi.WSGIContainer`. This is most useful to use both
Tornado ``RequestHandlers`` and WSGI in the same server. Typical
usage::
wsgi_app = tornado.wsgi.WSGIContainer(
django.core.handlers.wsgi.WSGIHandler())
@@ -1837,7 +1879,11 @@ class ChunkedTransferEncoding(OutputTransform):
def authenticated(method):
"""Decorate methods with this to require that the user be logged in."""
"""Decorate methods with this to require that the user be logged in.
If the user is not logged in, they will be redirected to the configured
`login url <RequestHandler.get_login_url>`.
"""
@functools.wraps(method)
def wrapper(self, *args, **kwargs):
if not self.current_user:
@@ -1858,7 +1904,7 @@ def authenticated(method):
class UIModule(object):
"""A UI re-usable, modular unit on a page.
"""A re-usable, modular UI unit on a page.
UI modules often execute additional queries, and they can include
additional CSS and JavaScript that will be included in the output
@@ -1985,21 +2031,19 @@ class TemplateModule(UIModule):
class URLSpec(object):
"""Specifies mappings between URLs and handlers."""
def __init__(self, pattern, handler_class, kwargs=None, name=None):
"""Creates a URLSpec.
"""Parameters:
Parameters:
* ``pattern``: Regular expression to be matched. Any groups
in the regex will be passed in to the handler's get/post/etc
methods as arguments.
pattern: Regular expression to be matched. Any groups in the regex
will be passed in to the handler's get/post/etc methods as
arguments.
* ``handler_class``: `RequestHandler` subclass to be invoked.
handler_class: RequestHandler subclass to be invoked.
* ``kwargs`` (optional): A dictionary of additional arguments
to be passed to the handler's constructor.
kwargs (optional): A dictionary of additional arguments to be passed
to the handler's constructor.
name (optional): A name for this handler. Used by
Application.reverse_url.
* ``name`` (optional): A name for this handler. Used by
`Application.reverse_url`.
"""
if not pattern.endswith('$'):
pattern += '$'
libs/tornado/websocket.py
+64 -44
View File
@@ -1,4 +1,4 @@
"""Server-side implementation of the WebSocket protocol.
"""Implementation of the WebSocket protocol.
`WebSockets <http://dev.w3.org/html5/websockets/>`_ allow for bidirectional
communication between the browser and server.
@@ -25,46 +25,44 @@ import base64
import collections
import functools
import hashlib
import logging
import os
import re
import socket
import struct
import time
import tornado.escape
import tornado.web
from tornado.concurrent import Future, return_future
from tornado.escape import utf8, to_unicode, native_str
from tornado.httputil import HTTPHeaders
from tornado.concurrent import Future
from tornado.escape import utf8, native_str
from tornado import httpclient
from tornado.ioloop import IOLoop
from tornado.iostream import IOStream, SSLIOStream
from tornado.log import gen_log, app_log
from tornado.netutil import Resolver
from tornado import simple_httpclient
from tornado.util import bytes_type
from tornado.util import bytes_type, unicode_type
try:
xrange # py2
except NameError:
xrange = range # py3
try:
import urlparse # py2
except ImportError:
import urllib.parse as urlparse # py3
class WebSocketError(Exception):
pass
class WebSocketHandler(tornado.web.RequestHandler):
"""Subclass this class to create a basic WebSocket handler.
Override on_message to handle incoming messages. You can also override
open and on_close to handle opened and closed connections.
Override `on_message` to handle incoming messages, and use
`write_message` to send messages to the client. You can also
override `open` and `on_close` to handle opened and closed
connections.
See http://dev.w3.org/html5/websockets/ for details on the
JavaScript interface. The protocol is specified at
http://tools.ietf.org/html/rfc6455.
Here is an example Web Socket handler that echos back all received messages
Here is an example WebSocket handler that echos back all received messages
back to the client::
class EchoWebSocket(websocket.WebSocketHandler):
@@ -77,14 +75,15 @@ class WebSocketHandler(tornado.web.RequestHandler):
def on_close(self):
print "WebSocket closed"
Web Sockets are not standard HTTP connections. The "handshake" is HTTP,
but after the handshake, the protocol is message-based. Consequently,
most of the Tornado HTTP facilities are not available in handlers of this
type. The only communication methods available to you are write_message()
and close(). Likewise, your request handler class should
implement open() method rather than get() or post().
WebSockets are not standard HTTP connections. The "handshake" is
HTTP, but after the handshake, the protocol is
message-based. Consequently, most of the Tornado HTTP facilities
are not available in handlers of this type. The only communication
methods available to you are `write_message()`, `ping()`, and
`close()`. Likewise, your request handler class should implement
`open()` method rather than ``get()`` or ``post()``.
If you map the handler above to "/websocket" in your application, you can
If you map the handler above to ``/websocket`` in your application, you can
invoke it in JavaScript with::
var ws = new WebSocket("ws://localhost:8888/websocket");
@@ -211,6 +210,7 @@ class WebSocketHandler(tornado.web.RequestHandler):
Once the close handshake is successful the socket will be closed.
"""
self.ws_connection.close()
self.ws_connection = None
def allow_draft76(self):
"""Override to enable support for the older "draft76" protocol.
@@ -240,11 +240,10 @@ class WebSocketHandler(tornado.web.RequestHandler):
return "wss" if self.request.protocol == "https" else "ws"
def async_callback(self, callback, *args, **kwargs):
"""Wrap callbacks with this if they are used on asynchronous requests.
"""Obsolete - catches exceptions from the wrapped function.
Catches exceptions properly and closes this WebSocket if an exception
is uncaught. (Note that this is usually unnecessary thanks to
`tornado.stack_context`)
This function is normally unncecessary thanks to
`tornado.stack_context`.
"""
return self.ws_connection.async_callback(callback, *args, **kwargs)
@@ -398,8 +397,9 @@ class WebSocketProtocol76(WebSocketProtocol):
"""Processes the key headers and calculates their key value.
Raises ValueError when feed invalid key."""
# pyflakes complains about variable reuse if both of these lines use 'c'
number = int(''.join(c for c in key if c.isdigit()))
spaces = len([c for c in key if c.isspace()])
spaces = len([c2 for c2 in key if c2.isspace()])
try:
key_number = number // spaces
except (ValueError, ZeroDivisionError):
@@ -444,7 +444,7 @@ class WebSocketProtocol76(WebSocketProtocol):
if binary:
raise ValueError(
"Binary messages not supported by this version of websockets")
if isinstance(message, unicode):
if isinstance(message, unicode_type):
message = message.encode("utf-8")
assert isinstance(message, bytes_type)
self.stream.write(b"\x00" + message + b"\xff")
@@ -520,7 +520,6 @@ class WebSocketProtocol13(WebSocketProtocol):
return WebSocketProtocol13.compute_accept_value(
self.request.headers.get("Sec-Websocket-Key"))
def _accept_connection(self):
subprotocol_header = ''
subprotocols = self.request.headers.get("Sec-WebSocket-Protocol", '')
@@ -727,7 +726,8 @@ class WebSocketProtocol13(WebSocketProtocol):
self.stream.io_loop.time() + 5, self._abort)
class _WebSocketClientConnection(simple_httpclient._HTTPConnection):
class WebSocketClientConnection(simple_httpclient._HTTPConnection):
"""WebSocket client connection."""
def __init__(self, io_loop, request):
self.connect_future = Future()
self.read_future = None
@@ -738,19 +738,26 @@ class _WebSocketClientConnection(simple_httpclient._HTTPConnection):
scheme = {'ws': 'http', 'wss': 'https'}[scheme]
request.url = scheme + sep + rest
request.headers.update({
'Upgrade': 'websocket',
'Connection': 'Upgrade',
'Sec-WebSocket-Key': self.key,
'Sec-WebSocket-Version': '13',
})
'Upgrade': 'websocket',
'Connection': 'Upgrade',
'Sec-WebSocket-Key': self.key,
'Sec-WebSocket-Version': '13',
})
super(_WebSocketClientConnection, self).__init__(
io_loop, None, request, lambda: None, lambda response: None,
super(WebSocketClientConnection, self).__init__(
io_loop, None, request, lambda: None, self._on_http_response,
104857600, Resolver(io_loop=io_loop))
def _on_close(self):
self.on_message(None)
def _on_http_response(self, response):
if not self.connect_future.done():
if response.error:
self.connect_future.set_exception(response.error)
else:
self.connect_future.set_exception(WebSocketError(
"Non-websocket response"))
def _handle_1xx(self, code):
assert code == 101
@@ -769,9 +776,17 @@ class _WebSocketClientConnection(simple_httpclient._HTTPConnection):
self.connect_future.set_result(self)
def write_message(self, message, binary=False):
"""Sends a message to the WebSocket server."""
self.protocol.write_message(message, binary)
def read_message(self, callback=None):
"""Reads a message from the WebSocket server.
Returns a future whose result is the message, or None
if the connection is closed. If a callback argument
is given it will be called with the future when it is
ready.
"""
assert self.read_future is None
future = Future()
if self.read_queue:
@@ -793,13 +808,18 @@ class _WebSocketClientConnection(simple_httpclient._HTTPConnection):
pass
def WebSocketConnect(url, io_loop=None, callback=None):
def websocket_connect(url, io_loop=None, callback=None, connect_timeout=None):
"""Client-side websocket support.
Takes a url and returns a Future whose result is a
`WebSocketClientConnection`.
"""
if io_loop is None:
io_loop = IOLoop.instance()
request = simple_httpclient.HTTPRequest(url)
request = simple_httpclient._RequestProxy(
request, simple_httpclient.HTTPRequest._DEFAULTS)
conn = _WebSocketClientConnection(io_loop, request)
io_loop = IOLoop.current()
request = httpclient.HTTPRequest(url, connect_timeout=connect_timeout)
request = httpclient._RequestProxy(
request, httpclient.HTTPRequest._DEFAULTS)
conn = WebSocketClientConnection(io_loop, request)
if callback is not None:
io_loop.add_future(conn.connect_future, callback)
return conn.connect_future
libs/tornado/wsgi.py
+15 -13
View File
@@ -82,11 +82,11 @@ else:
class WSGIApplication(web.Application):
"""A WSGI equivalent of `tornado.web.Application`.
WSGIApplication is very similar to web.Application, except no
asynchronous methods are supported (since WSGI does not support
non-blocking requests properly). If you call self.flush() or other
asynchronous methods in your request handlers running in a
WSGIApplication, we throw an exception.
`WSGIApplication` is very similar to `tornado.web.Application`,
except no asynchronous methods are supported (since WSGI does not
support non-blocking requests properly). If you call
``self.flush()`` or other asynchronous methods in your request
handlers running in a `WSGIApplication`, we throw an exception.
Example usage::
@@ -105,13 +105,15 @@ class WSGIApplication(web.Application):
server = wsgiref.simple_server.make_server('', 8888, application)
server.serve_forever()
See the 'appengine' demo for an example of using this module to run
a Tornado app on Google AppEngine.
See the `appengine demo
<https://github.com/facebook/tornado/tree/master/demos/appengine>`_
for an example of using this module to run a Tornado app on Google
App Engine.
Since no asynchronous methods are available for WSGI applications, the
httpclient and auth modules are both not available for WSGI applications.
We support the same interface, but handlers running in a WSGIApplication
do not support flush() or asynchronous methods.
WSGI applications use the same `.RequestHandler` class, but not
``@asynchronous`` methods or ``flush()``. This means that it is
not possible to use `.AsyncHTTPClient`, or the `tornado.auth` or
`tornado.websocket` modules.
"""
def __init__(self, handlers=None, default_host="", **settings):
web.Application.__init__(self, handlers, default_host, transforms=[],
@@ -134,7 +136,7 @@ class WSGIApplication(web.Application):
class HTTPRequest(object):
"""Mimics `tornado.httpserver.HTTPRequest` for WSGI applications."""
def __init__(self, environ):
"""Parses the given WSGI environ to construct the request."""
"""Parses the given WSGI environment to construct the request."""
self.method = environ["REQUEST_METHOD"]
self.path = urllib_parse.quote(from_wsgi_str(environ.get("SCRIPT_NAME", "")))
self.path += urllib_parse.quote(from_wsgi_str(environ.get("PATH_INFO", "")))
@@ -206,7 +208,7 @@ class HTTPRequest(object):
class WSGIContainer(object):
r"""Makes a WSGI-compatible function runnable on Tornado's HTTP server.
Wrap a WSGI function in a WSGIContainer and pass it to HTTPServer to
Wrap a WSGI function in a `WSGIContainer` and pass it to `.HTTPServer` to
run it. For example::
def simple_app(environ, start_response):