diff --git a/couchpotato/core/_base/_core/main.py b/couchpotato/core/_base/_core/main.py index c91140fa..d2b6e2d3 100644 --- a/couchpotato/core/_base/_core/main.py +++ b/couchpotato/core/_base/_core/main.py @@ -79,7 +79,7 @@ class Core(Plugin): def shutdown(): self.initShutdown() - IOLoop.instance().add_callback(shutdown) + IOLoop.current().add_callback(shutdown) return 'shutdown' @@ -89,7 +89,7 @@ class Core(Plugin): def restart(): self.initShutdown(restart = True) - IOLoop.instance().add_callback(restart) + IOLoop.current().add_callback(restart) return 'restarting' @@ -128,7 +128,7 @@ class Core(Plugin): log.debug('Save to shutdown/restart') try: - IOLoop.instance().stop() + IOLoop.current().stop() except RuntimeError: pass except: diff --git a/couchpotato/runner.py b/couchpotato/runner.py index 0ae14ce0..26522178 100644 --- a/couchpotato/runner.py +++ b/couchpotato/runner.py @@ -241,7 +241,8 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En from tornado.ioloop import IOLoop web_container = WSGIContainer(app) web_container._log = _log - loop = IOLoop.instance() + loop = IOLoop.current() + application = Application([ (r'%s/api/%s/nonblock/(.*)/' % (url_base, api_key), NonBlockHandler), diff --git a/libs/tornado/__init__.py b/libs/tornado/__init__.py index 609e2c05..68434e18 100755 --- a/libs/tornado/__init__.py +++ b/libs/tornado/__init__.py @@ -23,7 +23,7 @@ from __future__ import absolute_import, division, print_function, with_statement # version_info is a four-tuple for programmatic comparison. The first # three numbers are the components of the version number. The fourth # is zero for an official release, positive for a development branch, -# or negative for a release candidate (after the base version number -# has been incremented) -version = "2.4.post3" -version_info = (2, 4, 0, 3) +# or negative for a release candidate or beta (after the base version +# number has been incremented) +version = "3.1.dev2" +version_info = (3, 1, 0, -99) diff --git a/libs/tornado/auth.py b/libs/tornado/auth.py index 0ff32cb2..df95884b 100755 --- a/libs/tornado/auth.py +++ b/libs/tornado/auth.py @@ -14,15 +14,19 @@ # License for the specific language governing permissions and limitations # under the License. -"""Implementations of various third-party authentication schemes. +"""This module contains implementations of various third-party +authentication schemes. -All the classes in this file are class Mixins designed to be used with -web.py RequestHandler classes. The primary methods for each service are -authenticate_redirect(), authorize_redirect(), and get_authenticated_user(). -The former should be called to redirect the user to, e.g., the OpenID -authentication page on the third party service, and the latter should -be called upon return to get the user data from the data returned by -the third party service. +All the classes in this file are class mixins designed to be used with +the `tornado.web.RequestHandler` class. They are used in two ways: + +* On a login handler, use methods such as ``authenticate_redirect()``, + ``authorize_redirect()``, and ``get_authenticated_user()`` to + establish the user's identity and store authentication tokens to your + database and/or cookies. +* In non-login handlers, use methods such as ``facebook_request()`` + or ``twitter_request()`` to use the authentication tokens to make + requests to the respective services. They all take slightly different arguments due to the fact all these services implement authentication and authorization slightly differently. @@ -30,18 +34,16 @@ See the individual service classes below for complete documentation. Example usage for Google OpenID:: - class GoogleHandler(tornado.web.RequestHandler, tornado.auth.GoogleMixin): + class GoogleLoginHandler(tornado.web.RequestHandler, + tornado.auth.GoogleMixin): @tornado.web.asynchronous + @tornado.gen.coroutine def get(self): if self.get_argument("openid.mode", None): - self.get_authenticated_user(self.async_callback(self._on_auth)) - return - self.authenticate_redirect() - - def _on_auth(self, user): - if not user: - raise tornado.web.HTTPError(500, "Google auth failed") - # Save the user with, e.g., set_secure_cookie() + user = yield self.get_authenticated_user() + # Save the user with e.g. set_secure_cookie() + else: + self.authenticate_redirect() """ from __future__ import absolute_import, division, print_function, with_statement @@ -72,9 +74,11 @@ try: except ImportError: import urllib as urllib_parse # py2 + class AuthError(Exception): pass + def _auth_future_to_callback(callback, future): try: result = future.result() @@ -83,6 +87,7 @@ def _auth_future_to_callback(callback, future): result = None callback(result) + def _auth_return_future(f): """Similar to tornado.concurrent.return_future, but uses the auth module's legacy callback interface. @@ -91,6 +96,7 @@ def _auth_return_future(f): inside the function will actually be a future. """ replacer = ArgReplacer(f, 'callback') + @functools.wraps(f) def wrapper(*args, **kwargs): future = Future() @@ -102,17 +108,23 @@ def _auth_return_future(f): return future return wrapper + class OpenIdMixin(object): """Abstract implementation of OpenID and Attribute Exchange. - See GoogleMixin below for example implementations. + See `GoogleMixin` below for a customized example (which also + includes OAuth support). + + Class attributes: + + * ``_OPENID_ENDPOINT``: the identity provider's URI. """ def authenticate_redirect(self, callback_uri=None, ax_attrs=["name", "email", "language", "username"]): - """Returns the authentication URL for this service. + """Redirects to the authentication URL for this service. After authentication, the service will redirect back to the given - callback URI. + callback URI with additional parameters including ``openid.mode``. We request the given attributes for the authenticated user by default (name, email, language, and username). If you don't need @@ -128,8 +140,12 @@ class OpenIdMixin(object): """Fetches the authenticated user data upon redirect. This method should be called by the handler that receives the - redirect from the authenticate_redirect() or authorize_redirect() - methods. + redirect from the `authenticate_redirect()` method (which is + often the same as the one that calls it; in that case you would + call `get_authenticated_user` if the ``openid.mode`` parameter + is present and `authenticate_redirect` if it is not). + + The result of this method will generally be used to set a cookie. """ # Verify the OpenID response via direct request to the OP args = dict((k, v[-1]) for k, v in self.request.arguments.items()) @@ -192,8 +208,8 @@ class OpenIdMixin(object): def _on_authentication_verified(self, future, response): if response.error or b"is_valid:true" not in response.body: future.set_exception(AuthError( - "Invalid OpenID response: %s" % (response.error or - response.body))) + "Invalid OpenID response: %s" % (response.error or + response.body))) return # Make sure we got back at least an email from attribute exchange @@ -250,32 +266,44 @@ class OpenIdMixin(object): future.set_result(user) def get_auth_http_client(self): - """Returns the AsyncHTTPClient instance to be used for auth requests. + """Returns the `.AsyncHTTPClient` instance to be used for auth requests. - May be overridden by subclasses to use an http client other than + May be overridden by subclasses to use an HTTP client other than the default. """ return httpclient.AsyncHTTPClient() class OAuthMixin(object): - """Abstract implementation of OAuth. + """Abstract implementation of OAuth 1.0 and 1.0a. - See TwitterMixin and FriendFeedMixin below for example implementations. + See `TwitterMixin` and `FriendFeedMixin` below for example implementations, + or `GoogleMixin` for an OAuth/OpenID hybrid. + + Class attributes: + + * ``_OAUTH_AUTHORIZE_URL``: The service's OAuth authorization url. + * ``_OAUTH_ACCESS_TOKEN_URL``: The service's OAuth access token url. + * ``_OAUTH_VERSION``: May be either "1.0" or "1.0a". + * ``_OAUTH_NO_CALLBACKS``: Set this to True if the service requires + advance registration of callbacks. + + Subclasses must also override the `_oauth_get_user_future` and + `_oauth_consumer_token` methods. """ def authorize_redirect(self, callback_uri=None, extra_params=None, http_client=None): """Redirects the user to obtain OAuth authorization for this service. - Twitter and FriendFeed both require that you register a Callback - URL with your application. You should call this method to log the - user in, and then call get_authenticated_user() in the handler - you registered as your Callback URL to complete the authorization - process. + The ``callback_uri`` may be omitted if you have previously + registered a callback URI with the third-party service. For some + sevices (including Twitter and Friendfeed), you must use a + previously-registered callback URI and cannot specify a callback + via this method. - This method sets a cookie called _oauth_request_token which is - subsequently used (and cleared) in get_authenticated_user for + This method sets a cookie called ``_oauth_request_token`` which is + subsequently used (and cleared) in `get_authenticated_user` for security purposes. """ if callback_uri and getattr(self, "_OAUTH_NO_CALLBACKS", False): @@ -299,15 +327,15 @@ class OAuthMixin(object): @_auth_return_future def get_authenticated_user(self, callback, http_client=None): - """Gets the OAuth authorized user and access token on callback. - - This method should be called from the handler for your registered - OAuth Callback URL to complete the registration process. We call - callback with the authenticated user, which in addition to standard - attributes like 'name' includes the 'access_key' attribute, which - contains the OAuth access you can use to make authorized requests - to this service on behalf of the user. + """Gets the OAuth authorized user and access token. + This method should be called from the handler for your + OAuth callback URL to complete the registration process. We run the + callback with the authenticated user dictionary. This dictionary + will contain an ``access_key`` which can be used to make authorized + requests to this service on behalf of the user. The dictionary will + also contain other fields such as ``name``, depending on the service + used. """ future = callback request_key = escape.utf8(self.get_argument("oauth_token")) @@ -315,13 +343,13 @@ class OAuthMixin(object): request_cookie = self.get_cookie("_oauth_request_token") if not request_cookie: future.set_exception(AuthError( - "Missing OAuth request token cookie")) + "Missing OAuth request token cookie")) return self.clear_cookie("_oauth_request_token") cookie_key, cookie_secret = [base64.b64decode(escape.utf8(i)) for i in request_cookie.split("|")] if cookie_key != request_key: future.set_exception(AuthError( - "Request token does not match cookie")) + "Request token does not match cookie")) return token = dict(key=cookie_key, secret=cookie_secret) if oauth_verifier: @@ -339,7 +367,7 @@ class OAuthMixin(object): oauth_signature_method="HMAC-SHA1", oauth_timestamp=str(int(time.time())), oauth_nonce=escape.to_basestring(binascii.b2a_hex(uuid.uuid4().bytes)), - oauth_version=getattr(self, "_OAUTH_VERSION", "1.0a"), + oauth_version="1.0", ) if getattr(self, "_OAUTH_VERSION", "1.0a") == "1.0a": if callback_uri == "oob": @@ -360,8 +388,8 @@ class OAuthMixin(object): if response.error: raise Exception("Could not get request token") request_token = _oauth_parse_response(response.body) - data = (base64.b64encode(request_token["key"]) + b"|" + - base64.b64encode(request_token["secret"])) + data = (base64.b64encode(escape.utf8(request_token["key"])) + b"|" + + base64.b64encode(escape.utf8(request_token["secret"]))) self.set_cookie("_oauth_request_token", data) args = dict(oauth_token=request_token["key"]) if callback_uri == "oob": @@ -381,7 +409,7 @@ class OAuthMixin(object): oauth_signature_method="HMAC-SHA1", oauth_timestamp=str(int(time.time())), oauth_nonce=escape.to_basestring(binascii.b2a_hex(uuid.uuid4().bytes)), - oauth_version=getattr(self, "_OAUTH_VERSION", "1.0a"), + oauth_version="1.0", ) if "verifier" in request_token: args["oauth_verifier"] = request_token["verifier"] @@ -405,8 +433,29 @@ class OAuthMixin(object): self._oauth_get_user_future(access_token).add_done_callback( self.async_callback(self._on_oauth_get_user, access_token, future)) + def _oauth_consumer_token(self): + """Subclasses must override this to return their OAuth consumer keys. + + The return value should be a `dict` with keys ``key`` and ``secret``. + """ + raise NotImplementedError() + @return_future def _oauth_get_user_future(self, access_token, callback): + """Subclasses must override this to get basic information about the + user. + + Should return a `.Future` whose result is a dictionary + containing information about the user, which may have been + retrieved by using ``access_token`` to make a request to the + service. + + The access token will be added to the returned dictionary to make + the result of `get_authenticated_user`. + + For backwards compatibility, the callback-based ``_oauth_get_user`` + method is also supported. + """ # By default, call the old-style _oauth_get_user, but new code # should override this method instead. self._oauth_get_user(access_token, callback) @@ -439,7 +488,7 @@ class OAuthMixin(object): oauth_signature_method="HMAC-SHA1", oauth_timestamp=str(int(time.time())), oauth_nonce=escape.to_basestring(binascii.b2a_hex(uuid.uuid4().bytes)), - oauth_version=getattr(self, "_OAUTH_VERSION", "1.0a"), + oauth_version="1.0", ) args = {} args.update(base_args) @@ -450,30 +499,38 @@ class OAuthMixin(object): else: signature = _oauth_signature(consumer_token, method, url, args, access_token) - base_args["oauth_signature"] = signature + base_args["oauth_signature"] = escape.to_basestring(signature) return base_args def get_auth_http_client(self): - """Returns the AsyncHTTPClient instance to be used for auth requests. + """Returns the `.AsyncHTTPClient` instance to be used for auth requests. - May be overridden by subclasses to use an http client other than + May be overridden by subclasses to use an HTTP client other than the default. """ return httpclient.AsyncHTTPClient() class OAuth2Mixin(object): - """Abstract implementation of OAuth v 2.""" + """Abstract implementation of OAuth 2.0. + + See `FacebookGraphMixin` below for an example implementation. + + Class attributes: + + * ``_OAUTH_AUTHORIZE_URL``: The service's authorization url. + * ``_OAUTH_ACCESS_TOKEN_URL``: The service's access token url. + """ def authorize_redirect(self, redirect_uri=None, client_id=None, client_secret=None, extra_params=None): """Redirects the user to obtain OAuth authorization for this service. - Some providers require that you register a Callback - URL with your application. You should call this method to log the - user in, and then call get_authenticated_user() in the handler - you registered as your Callback URL to complete the authorization - process. + Some providers require that you register a redirect URL with + your application instead of passing one via this method. You + should call this method to log the user in, and then call + ``get_authenticated_user`` in the handler for your + redirect URL to complete the authorization process. """ args = { "redirect_uri": redirect_uri, @@ -503,35 +560,30 @@ class TwitterMixin(OAuthMixin): """Twitter OAuth authentication. To authenticate with Twitter, register your application with - Twitter at http://twitter.com/apps. Then copy your Consumer Key and - Consumer Secret to the application settings 'twitter_consumer_key' and - 'twitter_consumer_secret'. Use this Mixin on the handler for the URL - you registered as your application's Callback URL. + Twitter at http://twitter.com/apps. Then copy your Consumer Key + and Consumer Secret to the application + `~tornado.web.Application.settings` ``twitter_consumer_key`` and + ``twitter_consumer_secret``. Use this mixin on the handler for the + URL you registered as your application's callback URL. - When your application is set up, you can use this Mixin like this + When your application is set up, you can use this mixin like this to authenticate the user with Twitter and get access to their stream:: - class TwitterHandler(tornado.web.RequestHandler, - tornado.auth.TwitterMixin): + class TwitterLoginHandler(tornado.web.RequestHandler, + tornado.auth.TwitterMixin): @tornado.web.asynchronous + @tornado.gen.coroutine def get(self): if self.get_argument("oauth_token", None): - self.get_authenticated_user(self.async_callback(self._on_auth)) - return - self.authorize_redirect() + user = yield self.get_authenticated_user() + # Save the user using e.g. set_secure_cookie() + else: + self.authorize_redirect() - def _on_auth(self, user): - if not user: - raise tornado.web.HTTPError(500, "Twitter auth failed") - # Save the user using, e.g., set_secure_cookie() - - The user object returned by get_authenticated_user() includes the - attributes 'username', 'name', and all of the custom Twitter user - attributes describe at - http://apiwiki.twitter.com/Twitter-REST-API-Method%3A-users%C2%A0show - in addition to 'access_token'. You should save the access token with - the user; it is required to make requests on behalf of the user later - with twitter_request(). + The user object returned by `~OAuthMixin.get_authenticated_user` + includes the attributes ``username``, ``name``, ``access_token``, + and all of the custom Twitter user attributes described at + https://dev.twitter.com/docs/api/1.1/get/users/show """ _OAUTH_REQUEST_TOKEN_URL = "http://api.twitter.com/oauth/request_token" _OAUTH_ACCESS_TOKEN_URL = "http://api.twitter.com/oauth/access_token" @@ -541,7 +593,8 @@ class TwitterMixin(OAuthMixin): _TWITTER_BASE_URL = "http://api.twitter.com/1" def authenticate_redirect(self, callback_uri=None): - """Just like authorize_redirect(), but auto-redirects if authorized. + """Just like `~OAuthMixin.authorize_redirect`, but + auto-redirects if authorized. This is generally the right interface to use if you are using Twitter for single-sign on. @@ -553,35 +606,33 @@ class TwitterMixin(OAuthMixin): @_auth_return_future def twitter_request(self, path, callback=None, access_token=None, post_args=None, **args): - """Fetches the given API path, e.g., "/statuses/user_timeline/btaylor" + """Fetches the given API path, e.g., ``/statuses/user_timeline/btaylor`` - The path should not include the format (we automatically append - ".json" and parse the JSON output). + The path should not include the format or API version number. + (we automatically use JSON format and API version 1). - If the request is a POST, post_args should be provided. Query + If the request is a POST, ``post_args`` should be provided. Query string arguments should be given as keyword arguments. - All the Twitter methods are documented at - http://apiwiki.twitter.com/Twitter-API-Documentation. + All the Twitter methods are documented at http://dev.twitter.com/ - Many methods require an OAuth access token which you can obtain - through authorize_redirect() and get_authenticated_user(). The - user returned through that process includes an 'access_token' - attribute that can be used to make authenticated requests via - this method. Example usage:: + Many methods require an OAuth access token which you can + obtain through `~OAuthMixin.authorize_redirect` and + `~OAuthMixin.get_authenticated_user`. The user returned through that + process includes an 'access_token' attribute that can be used + to make authenticated requests via this method. Example + usage:: class MainHandler(tornado.web.RequestHandler, tornado.auth.TwitterMixin): @tornado.web.authenticated @tornado.web.asynchronous + @tornado.gen.coroutine def get(self): - self.twitter_request( + new_entry = yield self.twitter_request( "/statuses/update", post_args={"status": "Testing Tornado Web Server"}, - access_token=user["access_token"], - callback=self.async_callback(self._on_post)) - - def _on_post(self, new_entry): + access_token=self.current_user["access_token"]) if not new_entry: # Call failed; perhaps missing permission? self.authorize_redirect() @@ -617,8 +668,8 @@ class TwitterMixin(OAuthMixin): def _on_twitter_request(self, future, response): if response.error: future.set_exception(AuthError( - "Error response %s fetching %s" % (response.error, - response.request.url))) + "Error response %s fetching %s" % (response.error, + response.request.url))) return future.set_result(escape.json_decode(response.body)) @@ -629,49 +680,45 @@ class TwitterMixin(OAuthMixin): key=self.settings["twitter_consumer_key"], secret=self.settings["twitter_consumer_secret"]) - @return_future - @gen.engine - def _oauth_get_user_future(self, access_token, callback): + @gen.coroutine + def _oauth_get_user_future(self, access_token): user = yield self.twitter_request( - "/users/show/" + escape.native_str(access_token[b"screen_name"]), + "/users/show/" + escape.native_str(access_token["screen_name"]), access_token=access_token) if user: user["username"] = user["screen_name"] - callback(user) + raise gen.Return(user) class FriendFeedMixin(OAuthMixin): """FriendFeed OAuth authentication. To authenticate with FriendFeed, register your application with - FriendFeed at http://friendfeed.com/api/applications. Then - copy your Consumer Key and Consumer Secret to the application settings - 'friendfeed_consumer_key' and 'friendfeed_consumer_secret'. Use - this Mixin on the handler for the URL you registered as your - application's Callback URL. + FriendFeed at http://friendfeed.com/api/applications. Then copy + your Consumer Key and Consumer Secret to the application + `~tornado.web.Application.settings` ``friendfeed_consumer_key`` + and ``friendfeed_consumer_secret``. Use this mixin on the handler + for the URL you registered as your application's Callback URL. - When your application is set up, you can use this Mixin like this + When your application is set up, you can use this mixin like this to authenticate the user with FriendFeed and get access to their feed:: - class FriendFeedHandler(tornado.web.RequestHandler, - tornado.auth.FriendFeedMixin): + class FriendFeedLoginHandler(tornado.web.RequestHandler, + tornado.auth.FriendFeedMixin): @tornado.web.asynchronous + @tornado.gen.coroutine def get(self): if self.get_argument("oauth_token", None): - self.get_authenticated_user(self.async_callback(self._on_auth)) - return - self.authorize_redirect() + user = yield self.get_authenticated_user() + # Save the user using e.g. set_secure_cookie() + else: + self.authorize_redirect() - def _on_auth(self, user): - if not user: - raise tornado.web.HTTPError(500, "FriendFeed auth failed") - # Save the user using, e.g., set_secure_cookie() - - The user object returned by get_authenticated_user() includes the - attributes 'username', 'name', and 'description' in addition to - 'access_token'. You should save the access token with the user; + The user object returned by `~OAuthMixin.get_authenticated_user()` includes the + attributes ``username``, ``name``, and ``description`` in addition to + ``access_token``. You should save the access token with the user; it is required to make requests on behalf of the user later with - friendfeed_request(). + `friendfeed_request()`. """ _OAUTH_VERSION = "1.0" _OAUTH_REQUEST_TOKEN_URL = "https://friendfeed.com/account/oauth/request_token" @@ -685,30 +732,32 @@ class FriendFeedMixin(OAuthMixin): post_args=None, **args): """Fetches the given relative API path, e.g., "/bret/friends" - If the request is a POST, post_args should be provided. Query + If the request is a POST, ``post_args`` should be provided. Query string arguments should be given as keyword arguments. All the FriendFeed methods are documented at http://friendfeed.com/api/documentation. - Many methods require an OAuth access token which you can obtain - through authorize_redirect() and get_authenticated_user(). The - user returned through that process includes an 'access_token' - attribute that can be used to make authenticated requests via - this method. Example usage:: + Many methods require an OAuth access token which you can + obtain through `~OAuthMixin.authorize_redirect` and + `~OAuthMixin.get_authenticated_user`. The user returned + through that process includes an ``access_token`` attribute that + can be used to make authenticated requests via this + method. + + Example usage:: class MainHandler(tornado.web.RequestHandler, tornado.auth.FriendFeedMixin): @tornado.web.authenticated @tornado.web.asynchronous + @tornado.gen.coroutine def get(self): - self.friendfeed_request( + new_entry = yield self.friendfeed_request( "/entry", post_args={"body": "Testing Tornado Web Server"}, - access_token=self.current_user["access_token"], - callback=self.async_callback(self._on_post)) + access_token=self.current_user["access_token"]) - def _on_post(self, new_entry): if not new_entry: # Call failed; perhaps missing permission? self.authorize_redirect() @@ -739,8 +788,8 @@ class FriendFeedMixin(OAuthMixin): def _on_friendfeed_request(self, future, response): if response.error: future.set_exception(AuthError( - "Error response %s fetching %s" % (response.error, - response.request.url))) + "Error response %s fetching %s" % (response.error, + response.request.url))) return future.set_result(escape.json_decode(response.body)) @@ -751,9 +800,8 @@ class FriendFeedMixin(OAuthMixin): key=self.settings["friendfeed_consumer_key"], secret=self.settings["friendfeed_consumer_secret"]) - @return_future - @gen.engine - def _oauth_get_user(self, access_token, callback): + @gen.coroutine + def _oauth_get_user_future(self, access_token, callback): user = yield self.friendfeed_request( "/feedinfo/" + access_token["username"], include="id,name,description", access_token=access_token) @@ -770,26 +818,30 @@ class FriendFeedMixin(OAuthMixin): class GoogleMixin(OpenIdMixin, OAuthMixin): """Google Open ID / OAuth authentication. - No application registration is necessary to use Google for authentication - or to access Google resources on behalf of a user. To authenticate with - Google, redirect with authenticate_redirect(). On return, parse the - response with get_authenticated_user(). We send a dict containing the - values for the user, including 'email', 'name', and 'locale'. + No application registration is necessary to use Google for + authentication or to access Google resources on behalf of a user. + + Google implements both OpenID and OAuth in a hybrid mode. If you + just need the user's identity, use + `~OpenIdMixin.authenticate_redirect`. If you need to make + requests to Google on behalf of the user, use + `authorize_redirect`. On return, parse the response with + `~OpenIdMixin.get_authenticated_user`. We send a dict containing + the values for the user, including ``email``, ``name``, and + ``locale``. + Example usage:: - class GoogleHandler(tornado.web.RequestHandler, tornado.auth.GoogleMixin): + class GoogleLoginHandler(tornado.web.RequestHandler, + tornado.auth.GoogleMixin): @tornado.web.asynchronous + @tornado.gen.coroutine def get(self): if self.get_argument("openid.mode", None): - self.get_authenticated_user(self.async_callback(self._on_auth)) - return - self.authenticate_redirect() - - def _on_auth(self, user): - if not user: - raise tornado.web.HTTPError(500, "Google auth failed") - # Save the user with, e.g., set_secure_cookie() - + user = yield self.get_authenticated_user() + # Save the user with e.g. set_secure_cookie() + else: + self.authenticate_redirect() """ _OPENID_ENDPOINT = "https://www.google.com/accounts/o8/ud" _OAUTH_ACCESS_TOKEN_URL = "https://www.google.com/accounts/OAuthGetAccessToken" @@ -798,7 +850,8 @@ class GoogleMixin(OpenIdMixin, OAuthMixin): ax_attrs=["name", "email", "language", "username"]): """Authenticates and authorizes for the given Google resource. - Some of the available resources are: + Some of the available resources which can be used in the ``oauth_scope`` + argument are: * Gmail Contacts - http://www.google.com/m8/feeds/ * Calendar - http://www.google.com/calendar/feeds/ @@ -839,7 +892,7 @@ class GoogleMixin(OpenIdMixin, OAuthMixin): key=self.settings["google_consumer_key"], secret=self.settings["google_consumer_secret"]) - def _oauth_get_user_future(self, access_token, callback): + def _oauth_get_user_future(self, access_token): return OpenIdMixin.get_authenticated_user(self) @@ -853,9 +906,9 @@ class FacebookMixin(object): To authenticate with Facebook, register your application with Facebook at http://www.facebook.com/developers/apps.php. Then copy your API Key and Application Secret to the application settings - 'facebook_api_key' and 'facebook_secret'. + ``facebook_api_key`` and ``facebook_secret``. - When your application is set up, you can use this Mixin like this + When your application is set up, you can use this mixin like this to authenticate the user with Facebook:: class FacebookHandler(tornado.web.RequestHandler, @@ -872,11 +925,11 @@ class FacebookMixin(object): raise tornado.web.HTTPError(500, "Facebook auth failed") # Save the user using, e.g., set_secure_cookie() - The user object returned by get_authenticated_user() includes the - attributes 'facebook_uid' and 'name' in addition to session attributes - like 'session_key'. You should save the session key with the user; it is + The user object returned by `get_authenticated_user` includes the + attributes ``facebook_uid`` and ``name`` in addition to session attributes + like ``session_key``. You should save the session key with the user; it is required to make requests on behalf of the user later with - facebook_request(). + `facebook_request`. """ def authenticate_redirect(self, callback_uri=None, cancel_uri=None, extended_permissions=None): @@ -1029,9 +1082,9 @@ class FacebookMixin(object): return hashlib.md5(body).hexdigest() def get_auth_http_client(self): - """Returns the AsyncHTTPClient instance to be used for auth requests. + """Returns the `.AsyncHTTPClient` instance to be used for auth requests. - May be overridden by subclasses to use an http client other than + May be overridden by subclasses to use an HTTP client other than the default. """ return httpclient.AsyncHTTPClient() @@ -1043,6 +1096,7 @@ class FacebookGraphMixin(OAuth2Mixin): _OAUTH_AUTHORIZE_URL = "https://graph.facebook.com/oauth/authorize?" _OAUTH_NO_CALLBACKS = False + @_auth_return_future def get_authenticated_user(self, redirect_uri, client_id, client_secret, code, callback, extra_fields=None): """Handles the login for the Facebook user, returning a user object. @@ -1051,24 +1105,20 @@ class FacebookGraphMixin(OAuth2Mixin): class FacebookGraphLoginHandler(LoginHandler, tornado.auth.FacebookGraphMixin): @tornado.web.asynchronous + @tornado.gen.coroutine def get(self): if self.get_argument("code", False): - self.get_authenticated_user( - redirect_uri='/auth/facebookgraph/', - client_id=self.settings["facebook_api_key"], - client_secret=self.settings["facebook_secret"], - code=self.get_argument("code"), - callback=self.async_callback( - self._on_login)) - return - self.authorize_redirect(redirect_uri='/auth/facebookgraph/', - client_id=self.settings["facebook_api_key"], - extra_params={"scope": "read_stream,offline_access"}) - - def _on_login(self, user): - logging.error(user) - self.finish() - + user = yield self.get_authenticated_user( + redirect_uri='/auth/facebookgraph/', + client_id=self.settings["facebook_api_key"], + client_secret=self.settings["facebook_secret"], + code=self.get_argument("code")) + # Save the user with e.g. set_secure_cookie + else: + self.authorize_redirect( + redirect_uri='/auth/facebookgraph/', + client_id=self.settings["facebook_api_key"], + extra_params={"scope": "read_stream,offline_access"}) """ http = self.get_auth_http_client() args = { @@ -1088,10 +1138,9 @@ class FacebookGraphMixin(OAuth2Mixin): client_secret, callback, fields)) def _on_access_token(self, redirect_uri, client_id, client_secret, - callback, fields, response): + future, fields, response): if response.error: - gen_log.warning('Facebook auth error: %s' % str(response)) - callback(None) + future.set_exception(AuthError('Facebook auth error: %s' % str(response))) return args = escape.parse_qs_bytes(escape.native_str(response.body)) @@ -1103,14 +1152,14 @@ class FacebookGraphMixin(OAuth2Mixin): self.facebook_request( path="/me", callback=self.async_callback( - self._on_get_user_info, callback, session, fields), + self._on_get_user_info, future, session, fields), access_token=session["access_token"], fields=",".join(fields) ) - def _on_get_user_info(self, callback, session, fields, user): + def _on_get_user_info(self, future, session, fields, user): if user is None: - callback(None) + future.set_result(None) return fieldmap = {} @@ -1118,42 +1167,43 @@ class FacebookGraphMixin(OAuth2Mixin): fieldmap[field] = user.get(field) fieldmap.update({"access_token": session["access_token"], "session_expires": session.get("expires")}) - callback(fieldmap) + future.set_result(fieldmap) + @_auth_return_future def facebook_request(self, path, callback, access_token=None, post_args=None, **args): """Fetches the given relative API path, e.g., "/btaylor/picture" - If the request is a POST, post_args should be provided. Query + If the request is a POST, ``post_args`` should be provided. Query string arguments should be given as keyword arguments. An introduction to the Facebook Graph API can be found at http://developers.facebook.com/docs/api - Many methods require an OAuth access token which you can obtain - through authorize_redirect() and get_authenticated_user(). The - user returned through that process includes an 'access_token' - attribute that can be used to make authenticated requests via - this method. Example usage:: + Many methods require an OAuth access token which you can + obtain through `~OAuth2Mixin.authorize_redirect` and + `get_authenticated_user`. The user returned through that + process includes an ``access_token`` attribute that can be + used to make authenticated requests via this method. + + Example usage:: class MainHandler(tornado.web.RequestHandler, tornado.auth.FacebookGraphMixin): @tornado.web.authenticated @tornado.web.asynchronous + @tornado.gen.coroutine def get(self): - self.facebook_request( + new_entry = yield self.facebook_request( "/me/feed", post_args={"message": "I am posting from my Tornado application!"}, - access_token=self.current_user["access_token"], - callback=self.async_callback(self._on_post)) + access_token=self.current_user["access_token"]) - def _on_post(self, new_entry): if not new_entry: # Call failed; perhaps missing permission? self.authorize_redirect() return self.finish("Posted a message!") - """ url = "https://graph.facebook.com" + path all_args = {} @@ -1171,18 +1221,18 @@ class FacebookGraphMixin(OAuth2Mixin): else: http.fetch(url, callback=callback) - def _on_facebook_request(self, callback, response): + def _on_facebook_request(self, future, response): if response.error: - gen_log.warning("Error response %s fetching %s", response.error, - response.request.url) - callback(None) + future.set_exception(AuthError("Error response %s fetching %s", + response.error, response.request.url)) return - callback(escape.json_decode(response.body)) + + future.set_result(escape.json_decode(response.body)) def get_auth_http_client(self): - """Returns the AsyncHTTPClient instance to be used for auth requests. + """Returns the `.AsyncHTTPClient` instance to be used for auth requests. - May be overridden by subclasses to use an http client other than + May be overridden by subclasses to use an HTTP client other than the default. """ return httpclient.AsyncHTTPClient() @@ -1243,10 +1293,14 @@ def _oauth_escape(val): def _oauth_parse_response(body): - p = escape.parse_qs(body, keep_blank_values=False) - token = dict(key=p[b"oauth_token"][0], secret=p[b"oauth_token_secret"][0]) + # I can't find an officially-defined encoding for oauth responses and + # have never seen anyone use non-ascii. Leave the response in a byte + # string for python 2, and use utf8 on python 3. + body = escape.native_str(body) + p = urlparse.parse_qs(body, keep_blank_values=False) + token = dict(key=p["oauth_token"][0], secret=p["oauth_token_secret"][0]) # Add the extra parameters the Provider included to the token - special = (b"oauth_token", b"oauth_token_secret") + special = ("oauth_token", "oauth_token_secret") token.update((k, p[k][0]) for k in p if k not in special) return token diff --git a/libs/tornado/autoreload.py b/libs/tornado/autoreload.py index 4e424878..05754299 100755 --- a/libs/tornado/autoreload.py +++ b/libs/tornado/autoreload.py @@ -14,15 +14,24 @@ # License for the specific language governing permissions and limitations # under the License. -"""A module to automatically restart the server when a module is modified. +"""xAutomatically restart the server when a source file is modified. -Most applications should not call this module directly. Instead, pass the +Most applications should not access this module directly. Instead, pass the keyword argument ``debug=True`` to the `tornado.web.Application` constructor. This will enable autoreload mode as well as checking for changes to templates -and static resources. +and static resources. Note that restarting is a destructive operation +and any requests in progress will be aborted when the process restarts. -This module depends on IOLoop, so it will not work in WSGI applications -and Google AppEngine. It also will not work correctly when HTTPServer's +This module can also be used as a command-line wrapper around scripts +such as unit test runners. See the `main` method for details. + +The command-line wrapper and Application debug modes can be used together. +This combination is encouraged as the wrapper catches syntax errors and +other import-time failures, while debug mode catches changes once +the server has started. + +This module depends on `.IOLoop`, so it will not work in WSGI applications +and Google App Engine. It also will not work correctly when `.HTTPServer`'s multi-process mode is used. Reloading loses any Python interpreter command-line arguments (e.g. ``-u``) @@ -94,12 +103,8 @@ _io_loops = weakref.WeakKeyDictionary() def start(io_loop=None, check_time=500): - """Restarts the process automatically when a module is modified. - - We run on the I/O loop, and restarting is a destructive operation, - so will terminate any pending requests. - """ - io_loop = io_loop or ioloop.IOLoop.instance() + """Begins watching source files for changes using the given `.IOLoop`. """ + io_loop = io_loop or ioloop.IOLoop.current() if io_loop in _io_loops: return _io_loops[io_loop] = True @@ -137,8 +142,8 @@ def add_reload_hook(fn): Note that for open file and socket handles it is generally preferable to set the ``FD_CLOEXEC`` flag (using `fcntl` or - `tornado.platform.auto.set_close_exec`) instead of using a reload - hook to close them. + ``tornado.platform.auto.set_close_exec``) instead + of using a reload hook to close them. """ _reload_hooks.append(fn) diff --git a/libs/tornado/concurrent.py b/libs/tornado/concurrent.py index 59075a3a..15a039ca 100755 --- a/libs/tornado/concurrent.py +++ b/libs/tornado/concurrent.py @@ -13,12 +13,21 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. +"""Utilities for working with threads and ``Futures``. + +``Futures`` are a pattern for concurrent programming introduced in +Python 3.2 in the `concurrent.futures` package (this package has also +been backported to older versions of Python and can be installed with +``pip install futures``). Tornado will use `concurrent.futures.Future` if +it is available; otherwise it will use a compatible class defined in this +module. +""" from __future__ import absolute_import, division, print_function, with_statement import functools import sys -from tornado.stack_context import ExceptionStackContext +from tornado.stack_context import ExceptionStackContext, wrap from tornado.util import raise_exc_info, ArgReplacer try: @@ -26,10 +35,12 @@ try: except ImportError: futures = None + class ReturnValueIgnoredError(Exception): pass -class DummyFuture(object): + +class _DummyFuture(object): def __init__(self): self._done = False self._result = None @@ -87,50 +98,92 @@ class DummyFuture(object): self._callbacks = None if futures is None: - Future = DummyFuture + Future = _DummyFuture else: Future = futures.Future +class TracebackFuture(Future): + """Subclass of `Future` which can store a traceback with + exceptions. + + The traceback is automatically available in Python 3, but in the + Python 2 futures backport this information is discarded. + """ + def __init__(self): + super(TracebackFuture, self).__init__() + self.__exc_info = None + + def exc_info(self): + return self.__exc_info + + def set_exc_info(self, exc_info): + """Traceback-aware replacement for + `~concurrent.futures.Future.set_exception`. + """ + self.__exc_info = exc_info + self.set_exception(exc_info[1]) + + def result(self): + if self.__exc_info is not None: + raise_exc_info(self.__exc_info) + else: + return super(TracebackFuture, self).result() + + class DummyExecutor(object): def submit(self, fn, *args, **kwargs): - future = Future() + future = TracebackFuture() try: future.set_result(fn(*args, **kwargs)) - except Exception as e: - future.set_exception(e) + except Exception: + future.set_exc_info(sys.exc_info()) return future dummy_executor = DummyExecutor() def run_on_executor(fn): + """Decorator to run a synchronous method asynchronously on an executor. + + The decorated method may be called with a ``callback`` keyword + argument and returns a future. + """ @functools.wraps(fn) def wrapper(self, *args, **kwargs): callback = kwargs.pop("callback", None) future = self.executor.submit(fn, self, *args, **kwargs) if callback: - self.io_loop.add_future(future, callback) + self.io_loop.add_future(future, + lambda future: callback(future.result())) return future return wrapper +_NO_RESULT = object() + + def return_future(f): - """Decorator to make a function that returns via callback return a `Future`. + """Decorator to make a function that returns via callback return a + `Future`. The wrapped function should take a ``callback`` keyword argument and invoke it with one argument when it has finished. To signal failure, the function can simply raise an exception (which will be - captured by the `stack_context` and passed along to the `Future`). + captured by the `.StackContext` and passed along to the ``Future``). From the caller's perspective, the callback argument is optional. If one is given, it will be invoked when the function is complete - with the `Future` as an argument. If no callback is given, the caller - should use the `Future` to wait for the function to complete - (perhaps by yielding it in a `gen.engine` function, or passing it - to `IOLoop.add_future`). + with `Future.result()` as an argument. If the function fails, the + callback will not be run and an exception will be raised into the + surrounding `.StackContext`. + + If no callback is given, the caller should use the ``Future`` to + wait for the function to complete (perhaps by yielding it in a + `.gen.engine` function, or passing it to `.IOLoop.add_future`). Usage:: + @return_future def future_func(arg1, arg2, callback): # Do stuff (possibly asynchronous) @@ -142,19 +195,20 @@ def return_future(f): callback() Note that ``@return_future`` and ``@gen.engine`` can be applied to the - same function, provided ``@return_future`` appears first. + same function, provided ``@return_future`` appears first. However, + consider using ``@gen.coroutine`` instead of this combination. """ replacer = ArgReplacer(f, 'callback') + @functools.wraps(f) def wrapper(*args, **kwargs): - future = Future() - callback, args, kwargs = replacer.replace(future.set_result, - args, kwargs) - if callback is not None: - future.add_done_callback(callback) + future = TracebackFuture() + callback, args, kwargs = replacer.replace( + lambda value=_NO_RESULT: future.set_result(value), + args, kwargs) def handle_error(typ, value, tb): - future.set_exception(value) + future.set_exc_info((typ, value, tb)) return True exc_info = None with ExceptionStackContext(handle_error): @@ -172,9 +226,25 @@ def return_future(f): # go ahead and raise it to the caller directly without waiting # for them to inspect the Future. raise_exc_info(exc_info) + + # If the caller passed in a callback, schedule it to be called + # when the future resolves. It is important that this happens + # just before we return the future, or else we risk confusing + # stack contexts with multiple exceptions (one here with the + # immediate exception, and again when the future resolves and + # the callback triggers its exception by calling future.result()). + if callback is not None: + def run_callback(future): + result = future.result() + if result is _NO_RESULT: + callback() + else: + callback(future.result()) + future.add_done_callback(wrap(run_callback)) return future return wrapper + def chain_future(a, b): """Chain two futures together so that when one completes, so does the other. @@ -182,7 +252,10 @@ def chain_future(a, b): """ def copy(future): assert future is a - if a.exception() is not None: + if (isinstance(a, TracebackFuture) and isinstance(b, TracebackFuture) + and a.exc_info() is not None): + b.set_exc_info(a.exc_info()) + elif a.exception() is not None: b.set_exception(a.exception()) else: b.set_result(a.result()) diff --git a/libs/tornado/curl_httpclient.py b/libs/tornado/curl_httpclient.py index f46ea7b8..adc2314f 100755 --- a/libs/tornado/curl_httpclient.py +++ b/libs/tornado/curl_httpclient.py @@ -14,7 +14,7 @@ # License for the specific language governing permissions and limitations # under the License. -"""Blocking and non-blocking HTTP client implementations using pycurl.""" +"""Non-blocking HTTP client implementation using pycurl.""" from __future__ import absolute_import, division, print_function, with_statement @@ -30,7 +30,8 @@ from tornado.log import gen_log from tornado import stack_context from tornado.escape import utf8, native_str -from tornado.httpclient import HTTPRequest, HTTPResponse, HTTPError, AsyncHTTPClient, main, _RequestProxy +from tornado.httpclient import HTTPResponse, HTTPError, AsyncHTTPClient, main +from tornado.util import bytes_type try: from io import BytesIO # py3 @@ -171,7 +172,7 @@ class CurlAsyncHTTPClient(AsyncHTTPClient): # libcurl is ready. After each timeout, resync the scheduled # timeout with libcurl's current state. new_timeout = self._multi.timeout() - if new_timeout != -1: + if new_timeout >= 0: self._set_timeout(new_timeout) def _handle_force_timeout(self): @@ -319,7 +320,7 @@ def _curl_setup_request(curl, request, buffer, headers): write_function = request.streaming_callback else: write_function = buffer.write - if type(b'') is type(''): # py2 + if bytes_type is str: # py2 curl.setopt(pycurl.WRITEFUNCTION, write_function) else: # py3 # Upstream pycurl doesn't support py3, but ubuntu 12.10 includes @@ -410,7 +411,14 @@ def _curl_setup_request(curl, request, buffer, headers): if request.auth_username is not None: userpwd = "%s:%s" % (request.auth_username, request.auth_password or '') - curl.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_BASIC) + + if request.auth_mode is None or request.auth_mode == "basic": + curl.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_BASIC) + elif request.auth_mode == "digest": + curl.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_DIGEST) + else: + raise ValueError("Unsupported auth_mode %s" % request.auth_mode) + curl.setopt(pycurl.USERPWD, native_str(userpwd)) gen_log.debug("%s %s (username: %r)", request.method, request.url, request.auth_username) diff --git a/libs/tornado/escape.py b/libs/tornado/escape.py index 6d72532d..016fdade 100755 --- a/libs/tornado/escape.py +++ b/libs/tornado/escape.py @@ -28,9 +28,9 @@ import sys from tornado.util import bytes_type, unicode_type, basestring_type, u try: - from urllib.parse import parse_qs # py3 + from urllib.parse import parse_qs as _parse_qs # py3 except ImportError: - from urlparse import parse_qs # Python 2.6+ + from urlparse import parse_qs as _parse_qs # Python 2.6+ try: import htmlentitydefs # py2 @@ -54,7 +54,7 @@ _XHTML_ESCAPE_DICT = {'&': '&', '<': '<', '>': '>', '"': '"'} def xhtml_escape(value): - """Escapes a string so it is valid within XML or XHTML.""" + """Escapes a string so it is valid within HTML or XML.""" return _XHTML_ESCAPE_RE.sub(lambda match: _XHTML_ESCAPE_DICT[match.group(0)], to_basestring(value)) @@ -72,7 +72,7 @@ def json_encode(value): # the javscript. Some json libraries do this escaping by default, # although python's standard library does not, so we do it here. # http://stackoverflow.com/questions/1580647/json-why-are-forward-slashes-escaped - return json.dumps(recursive_unicode(value)).replace("`, + which use ``self.finish()`` in place of a callback argument. """ @functools.wraps(func) def wrapper(*args, **kwargs): @@ -116,20 +136,129 @@ def engine(func): if runner is not None: return runner.handle_exception(typ, value, tb) return False - with ExceptionStackContext(handle_exception) as deactivate: - gen = func(*args, **kwargs) - if isinstance(gen, types.GeneratorType): - runner = Runner(gen, deactivate) - runner.run() - return - assert gen is None, gen - deactivate() + with ExceptionStackContext(handle_exception): + try: + result = func(*args, **kwargs) + except (Return, StopIteration) as e: + result = getattr(e, 'value', None) + else: + if isinstance(result, types.GeneratorType): + def final_callback(value): + if value is not None: + raise ReturnValueIgnoredError( + "@gen.engine functions cannot return values: " + "%r" % (value,)) + assert value is None + runner = Runner(result, final_callback) + runner.run() + return + if result is not None: + raise ReturnValueIgnoredError( + "@gen.engine functions cannot return values: %r" % + (result,)) # no yield, so we're done return wrapper +def coroutine(func): + """Decorator for asynchronous generators. + + Any generator that yields objects from this module must be wrapped + in either this decorator or `engine`. These decorators only work + on functions that are already asynchronous. For + `~tornado.web.RequestHandler` :ref:`HTTP verb methods ` methods, this + means that both the `tornado.web.asynchronous` and + `tornado.gen.coroutine` decorators must be used (for proper + exception handling, ``asynchronous`` should come before + ``gen.coroutine``). + + Coroutines may "return" by raising the special exception + `Return(value) `. In Python 3.3+, it is also possible for + the function to simply use the ``return value`` statement (prior to + Python 3.3 generators were not allowed to also return values). + In all versions of Python a coroutine that simply wishes to exit + early may use the ``return`` statement without a value. + + Functions with this decorator return a `.Future`. Additionally, + they may be called with a ``callback`` keyword argument, which + will be invoked with the future's result when it resolves. If the + coroutine fails, the callback will not be run and an exception + will be raised into the surrounding `.StackContext`. The + ``callback`` argument is not visible inside the decorated + function; it is handled by the decorator itself. + + From the caller's perspective, ``@gen.coroutine`` is similar to + the combination of ``@return_future`` and ``@gen.engine``. + """ + @functools.wraps(func) + def wrapper(*args, **kwargs): + runner = None + future = TracebackFuture() + + if 'callback' in kwargs: + callback = kwargs.pop('callback') + IOLoop.current().add_future( + future, lambda future: callback(future.result())) + + def handle_exception(typ, value, tb): + try: + if runner is not None and runner.handle_exception(typ, value, tb): + return True + except Exception: + typ, value, tb = sys.exc_info() + future.set_exc_info((typ, value, tb)) + return True + with ExceptionStackContext(handle_exception): + try: + result = func(*args, **kwargs) + except (Return, StopIteration) as e: + result = getattr(e, 'value', None) + except Exception: + future.set_exc_info(sys.exc_info()) + return future + else: + if isinstance(result, types.GeneratorType): + def final_callback(value): + future.set_result(value) + runner = Runner(result, final_callback) + runner.run() + return future + future.set_result(result) + return future + return wrapper + + +class Return(Exception): + """Special exception to return a value from a `coroutine`. + + If this exception is raised, its value argument is used as the + result of the coroutine:: + + @gen.coroutine + def fetch_json(url): + response = yield AsyncHTTPClient().fetch(url) + raise gen.Return(json_decode(response.body)) + + In Python 3.3, this exception is no longer necessary: the ``return`` + statement can be used directly to return a value (previously + ``yield`` and ``return`` with a value could not be combined in the + same function). + + By analogy with the return statement, the value argument is optional, + but it is never necessary to ``raise gen.Return()``. The ``return`` + statement can be used with no arguments instead. + """ + def __init__(self, value=None): + super(Return, self).__init__() + self.value = value + + class YieldPoint(object): - """Base class for objects that may be yielded from the generator.""" + """Base class for objects that may be yielded from the generator. + + Applications do not normally need to use this class, but it may be + subclassed to provide additional yielding behavior. + """ def start(self, runner): """Called by the runner after the generator has yielded. @@ -195,7 +324,7 @@ class Wait(YieldPoint): class WaitAll(YieldPoint): - """Returns the results of multiple previous `Callbacks`. + """Returns the results of multiple previous `Callbacks `. The argument is a sequence of `Callback` keys, and the result is a list of results in the same order. @@ -291,7 +420,7 @@ class Multi(YieldPoint): def is_ready(self): finished = list(itertools.takewhile( - lambda i: i.is_ready(), self.unfinished_children)) + lambda i: i.is_ready(), self.unfinished_children)) self.unfinished_children.difference_update(finished) return not self.unfinished_children @@ -331,13 +460,13 @@ class Runner(object): def register_callback(self, key): """Adds ``key`` to the list of callbacks.""" if key in self.pending_callbacks: - raise KeyReuseError("key %r is already pending" % key) + raise KeyReuseError("key %r is already pending" % (key,)) self.pending_callbacks.add(key) def is_ready(self, key): """Returns true if a result is available for ``key``.""" if key not in self.pending_callbacks: - raise UnknownKeyError("key %r is not pending" % key) + raise UnknownKeyError("key %r is not pending" % (key,)) return key in self.results def set_result(self, key, result): @@ -374,7 +503,7 @@ class Runner(object): yielded = self.gen.throw(*exc_info) else: yielded = self.gen.send(next) - except StopIteration: + except (StopIteration, Return) as e: self.finished = True if self.pending_callbacks and not self.had_exception: # If we ran cleanly without waiting on all callbacks @@ -384,7 +513,7 @@ class Runner(object): raise LeakedCallbackError( "finished without waiting for callbacks %r" % self.pending_callbacks) - self.final_callback() + self.final_callback(getattr(e, 'value', None)) self.final_callback = None return except Exception: @@ -401,7 +530,8 @@ class Runner(object): except Exception: self.exc_info = sys.exc_info() else: - self.exc_info = (BadYieldError("yielded unknown object %r" % yielded),) + self.exc_info = (BadYieldError( + "yielded unknown object %r" % (yielded,)),) finally: self.running = False @@ -414,7 +544,7 @@ class Runner(object): else: result = None self.set_result(key, result) - return inner + return wrap(inner) def handle_exception(self, typ, value, tb): if not self.running and not self.finished: diff --git a/libs/tornado/httpclient.py b/libs/tornado/httpclient.py index 1b645504..551fd0b1 100755 --- a/libs/tornado/httpclient.py +++ b/libs/tornado/httpclient.py @@ -1,36 +1,35 @@ """Blocking and non-blocking HTTP client interfaces. This module defines a common interface shared by two implementations, -`simple_httpclient` and `curl_httpclient`. Applications may either +``simple_httpclient`` and ``curl_httpclient``. Applications may either instantiate their chosen implementation class directly or use the `AsyncHTTPClient` class from this module, which selects an implementation that can be overridden with the `AsyncHTTPClient.configure` method. -The default implementation is `simple_httpclient`, and this is expected +The default implementation is ``simple_httpclient``, and this is expected to be suitable for most users' needs. However, some applications may wish -to switch to `curl_httpclient` for reasons such as the following: +to switch to ``curl_httpclient`` for reasons such as the following: -* `curl_httpclient` has some features not found in `simple_httpclient`, +* ``curl_httpclient`` has some features not found in ``simple_httpclient``, including support for HTTP proxies and the ability to use a specified network interface. -* `curl_httpclient` is more likely to be compatible with sites that are +* ``curl_httpclient`` is more likely to be compatible with sites that are not-quite-compliant with the HTTP spec, or sites that use little-exercised features of HTTP. -* `simple_httpclient` only supports SSL on Python 2.6 and above. +* ``curl_httpclient`` is faster. -* `curl_httpclient` is faster +* ``curl_httpclient`` was the default prior to Tornado 2.0. -* `curl_httpclient` was the default prior to Tornado 2.0. - -Note that if you are using `curl_httpclient`, it is highly recommended that +Note that if you are using ``curl_httpclient``, it is highly recommended that you use a recent version of ``libcurl`` and ``pycurl``. Currently the minimum supported version is 7.18.2, and the recommended version is 7.21.1 or newer. """ from __future__ import absolute_import, division, print_function, with_statement +import functools import time import weakref @@ -52,15 +51,15 @@ class HTTPClient(object): try: response = http_client.fetch("http://www.google.com/") print response.body - except httpclient.HTTPError, e: + except httpclient.HTTPError as e: print "Error:", e + httpclient.close() """ def __init__(self, async_client_class=None, **kwargs): self._io_loop = IOLoop() if async_client_class is None: async_client_class = AsyncHTTPClient self._async_client = async_client_class(self._io_loop, **kwargs) - self._response = None self._closed = False def __del__(self): @@ -82,13 +81,8 @@ class HTTPClient(object): If an error occurs during the fetch, we raise an `HTTPError`. """ - def callback(response): - self._response = response - self._io_loop.stop() - self._async_client.fetch(request, callback, **kwargs) - self._io_loop.start() - response = self._response - self._response = None + response = self._io_loop.run_sync(functools.partial( + self._async_client.fetch, request, **kwargs)) response.rethrow() return response @@ -98,26 +92,23 @@ class AsyncHTTPClient(Configurable): Example usage:: - import ioloop - def handle_request(response): if response.error: print "Error:", response.error else: print response.body - ioloop.IOLoop.instance().stop() - http_client = httpclient.AsyncHTTPClient() + http_client = AsyncHTTPClient() http_client.fetch("http://www.google.com/", handle_request) - ioloop.IOLoop.instance().start() - The constructor for this class is magic in several respects: It actually - creates an instance of an implementation-specific subclass, and instances - are reused as a kind of pseudo-singleton (one per IOLoop). The keyword - argument force_instance=True can be used to suppress this singleton - behavior. Constructor arguments other than io_loop and force_instance - are deprecated. The implementation subclass as well as arguments to - its constructor can be set with the static method configure() + The constructor for this class is magic in several respects: It + actually creates an instance of an implementation-specific + subclass, and instances are reused as a kind of pseudo-singleton + (one per `.IOLoop`). The keyword argument ``force_instance=True`` + can be used to suppress this singleton behavior. Constructor + arguments other than ``io_loop`` and ``force_instance`` are + deprecated. The implementation subclass as well as arguments to + its constructor can be set with the static method `configure()` """ @classmethod def configurable_base(cls): @@ -136,7 +127,7 @@ class AsyncHTTPClient(Configurable): return getattr(cls, attr_name) def __new__(cls, io_loop=None, force_instance=False, **kwargs): - io_loop = io_loop or IOLoop.instance() + io_loop = io_loop or IOLoop.current() if io_loop in cls._async_clients() and not force_instance: return cls._async_clients()[io_loop] instance = super(AsyncHTTPClient, cls).__new__(cls, io_loop=io_loop, @@ -152,25 +143,29 @@ class AsyncHTTPClient(Configurable): self.defaults.update(defaults) def close(self): - """Destroys this http client, freeing any file descriptors used. + """Destroys this HTTP client, freeing any file descriptors used. Not needed in normal use, but may be helpful in unittests that create and destroy http clients. No other methods may be called - on the AsyncHTTPClient after close(). + on the `AsyncHTTPClient` after ``close()``. """ if self._async_clients().get(self.io_loop) is self: del self._async_clients()[self.io_loop] def fetch(self, request, callback=None, **kwargs): - """Executes a request, calling callback with an `HTTPResponse`. + """Executes a request, asynchronously returning an `HTTPResponse`. The request may be either a string URL or an `HTTPRequest` object. If it is a string, we construct an `HTTPRequest` using any additional kwargs: ``HTTPRequest(request, **kwargs)`` - If an error occurs during the fetch, the HTTPResponse given to the - callback has a non-None error attribute that contains the exception - encountered during the request. You can call response.rethrow() to - throw the exception (if any) in the callback. + This method returns a `.Future` whose result is an + `HTTPResponse`. The ``Future`` wil raise an `HTTPError` if + the request returned a non-200 response code. + + If a ``callback`` is given, it will be invoked with the `HTTPResponse`. + In the callback interface, `HTTPError` is not automatically raised. + Instead, you must check the response's ``error`` attribute or + call its `~HTTPResponse.rethrow` method. """ if not isinstance(request, HTTPRequest): request = HTTPRequest(url=request, **kwargs) @@ -182,6 +177,7 @@ class AsyncHTTPClient(Configurable): future = Future() if callback is not None: callback = stack_context.wrap(callback) + def handle_future(future): exc = future.exception() if isinstance(exc, HTTPError) and exc.response is not None: @@ -194,6 +190,7 @@ class AsyncHTTPClient(Configurable): response = future.result() self.io_loop.add_callback(callback, response) future.add_done_callback(handle_future) + def handle_response(response): if response.error: future.set_exception(response.error) @@ -207,19 +204,19 @@ class AsyncHTTPClient(Configurable): @classmethod def configure(cls, impl, **kwargs): - """Configures the AsyncHTTPClient subclass to use. + """Configures the `AsyncHTTPClient` subclass to use. - AsyncHTTPClient() actually creates an instance of a subclass. + ``AsyncHTTPClient()`` actually creates an instance of a subclass. This method may be called with either a class object or the - fully-qualified name of such a class (or None to use the default, - SimpleAsyncHTTPClient) + fully-qualified name of such a class (or ``None`` to use the default, + ``SimpleAsyncHTTPClient``) If additional keyword arguments are given, they will be passed to the constructor of each subclass instance created. The - keyword argument max_clients determines the maximum number of - simultaneous fetch() operations that can execute in parallel - on each IOLoop. Additional arguments may be supported depending - on the implementation class in use. + keyword argument ``max_clients`` determines the maximum number + of simultaneous `~AsyncHTTPClient.fetch()` operations that can + execute in parallel on each `.IOLoop`. Additional arguments + may be supported depending on the implementation class in use. Example:: @@ -245,7 +242,7 @@ class HTTPRequest(object): validate_cert=True) def __init__(self, url, method="GET", headers=None, body=None, - auth_username=None, auth_password=None, + auth_username=None, auth_password=None, auth_mode=None, connect_timeout=None, request_timeout=None, if_modified_since=None, follow_redirects=None, max_redirects=None, user_agent=None, use_gzip=None, @@ -256,59 +253,61 @@ class HTTPRequest(object): validate_cert=None, ca_certs=None, allow_ipv6=None, client_key=None, client_cert=None): - r"""Creates an `HTTPRequest`. - - All parameters except `url` are optional. + r"""All parameters except ``url`` are optional. :arg string url: URL to fetch :arg string method: HTTP method, e.g. "GET" or "POST" :arg headers: Additional HTTP headers to pass on the request :type headers: `~tornado.httputil.HTTPHeaders` or `dict` - :arg string auth_username: Username for HTTP "Basic" authentication - :arg string auth_password: Password for HTTP "Basic" authentication + :arg string auth_username: Username for HTTP authentication + :arg string auth_password: Password for HTTP authentication + :arg string auth_mode: Authentication mode; default is "basic". + Allowed values are implementation-defined; ``curl_httpclient`` + supports "basic" and "digest"; ``simple_httpclient`` only supports + "basic" :arg float connect_timeout: Timeout for initial connection in seconds :arg float request_timeout: Timeout for entire request in seconds - :arg datetime if_modified_since: Timestamp for ``If-Modified-Since`` - header + :arg if_modified_since: Timestamp for ``If-Modified-Since`` header + :type if_modified_since: `datetime` or `float` :arg bool follow_redirects: Should redirects be followed automatically or return the 3xx response? - :arg int max_redirects: Limit for `follow_redirects` + :arg int max_redirects: Limit for ``follow_redirects`` :arg string user_agent: String to send as ``User-Agent`` header :arg bool use_gzip: Request gzip encoding from the server :arg string network_interface: Network interface to use for request - :arg callable streaming_callback: If set, `streaming_callback` will + :arg callable streaming_callback: If set, ``streaming_callback`` will be run with each chunk of data as it is received, and - `~HTTPResponse.body` and `~HTTPResponse.buffer` will be empty in + ``HTTPResponse.body`` and ``HTTPResponse.buffer`` will be empty in the final response. - :arg callable header_callback: If set, `header_callback` will + :arg callable header_callback: If set, ``header_callback`` will be run with each header line as it is received (including the first line, e.g. ``HTTP/1.0 200 OK\r\n``, and a final line containing only ``\r\n``. All lines include the trailing newline - characters). `~HTTPResponse.headers` will be empty in the final + characters). ``HTTPResponse.headers`` will be empty in the final response. This is most useful in conjunction with - `streaming_callback`, because it's the only way to get access to + ``streaming_callback``, because it's the only way to get access to header data while the request is in progress. :arg callable prepare_curl_callback: If set, will be called with - a `pycurl.Curl` object to allow the application to make additional - `setopt` calls. + a ``pycurl.Curl`` object to allow the application to make additional + ``setopt`` calls. :arg string proxy_host: HTTP proxy hostname. To use proxies, - `proxy_host` and `proxy_port` must be set; `proxy_username` and - `proxy_pass` are optional. Proxies are currently only support - with `curl_httpclient`. + ``proxy_host`` and ``proxy_port`` must be set; ``proxy_username`` and + ``proxy_pass`` are optional. Proxies are currently only supported + with ``curl_httpclient``. :arg int proxy_port: HTTP proxy port :arg string proxy_username: HTTP proxy username :arg string proxy_password: HTTP proxy password - :arg bool allow_nonstandard_methods: Allow unknown values for `method` + :arg bool allow_nonstandard_methods: Allow unknown values for ``method`` argument? :arg bool validate_cert: For HTTPS requests, validate the server's certificate? :arg string ca_certs: filename of CA certificates in PEM format, - or None to use defaults. Note that in `curl_httpclient`, if - any request uses a custom `ca_certs` file, they all must (they - don't have to all use the same `ca_certs`, but it's not possible - to mix requests with ca_certs and requests that use the defaults. + or None to use defaults. Note that in ``curl_httpclient``, if + any request uses a custom ``ca_certs`` file, they all must (they + don't have to all use the same ``ca_certs``, but it's not possible + to mix requests with ``ca_certs`` and requests that use the defaults. :arg bool allow_ipv6: Use IPv6 when available? Default is false in - `simple_httpclient` and true in `curl_httpclient` + ``simple_httpclient`` and true in ``curl_httpclient`` :arg string client_key: Filename for client SSL key, if any :arg string client_cert: Filename for client SSL certificate, if any """ @@ -327,6 +326,7 @@ class HTTPRequest(object): self.body = utf8(body) self.auth_username = auth_username self.auth_password = auth_password + self.auth_mode = auth_mode self.connect_timeout = connect_timeout self.request_timeout = request_timeout self.follow_redirects = follow_redirects @@ -356,29 +356,32 @@ class HTTPResponse(object): * code: numeric HTTP status code, e.g. 200 or 404 * reason: human-readable reason phrase describing the status code - (with curl_httpclient, this is a default value rather than the - server's actual response) + (with curl_httpclient, this is a default value rather than the + server's actual response) - * headers: httputil.HTTPHeaders object + * headers: `tornado.httputil.HTTPHeaders` object - * buffer: cStringIO object for response body + * buffer: ``cStringIO`` object for response body - * body: response body as string (created on demand from self.buffer) + * body: response body as string (created on demand from ``self.buffer``) * error: Exception object, if any * request_time: seconds from request start to finish * time_info: dictionary of diagnostic timing information from the request. - Available data are subject to change, but currently uses timings - available from http://curl.haxx.se/libcurl/c/curl_easy_getinfo.html, - plus 'queue', which is the delay (if any) introduced by waiting for - a slot under AsyncHTTPClient's max_clients setting. + Available data are subject to change, but currently uses timings + available from http://curl.haxx.se/libcurl/c/curl_easy_getinfo.html, + plus ``queue``, which is the delay (if any) introduced by waiting for + a slot under `AsyncHTTPClient`'s ``max_clients`` setting. """ def __init__(self, request, code, headers=None, buffer=None, effective_url=None, error=None, request_time=None, time_info=None, reason=None): - self.request = request + if isinstance(request, _RequestProxy): + self.request = request.request + else: + self.request = request self.code = code self.reason = reason or httputil.responses.get(code, "Unknown") if headers is not None: @@ -426,13 +429,13 @@ class HTTPError(Exception): Attributes: - code - HTTP error integer error code, e.g. 404. Error code 599 is - used when no HTTP response was received, e.g. for a timeout. + * ``code`` - HTTP error integer error code, e.g. 404. Error code 599 is + used when no HTTP response was received, e.g. for a timeout. - response - HTTPResponse object, if any. + * ``response`` - `HTTPResponse` object, if any. - Note that if follow_redirects is False, redirects become HTTPErrors, - and you can look at error.response.headers['Location'] to see the + Note that if ``follow_redirects`` is False, redirects become HTTPErrors, + and you can look at ``error.response.headers['Location']`` to see the destination of the redirect. """ def __init__(self, code, message=None, response=None): diff --git a/libs/tornado/httpserver.py b/libs/tornado/httpserver.py index 24d5e6b3..ef36e6bd 100755 --- a/libs/tornado/httpserver.py +++ b/libs/tornado/httpserver.py @@ -34,14 +34,15 @@ from tornado.escape import native_str, parse_qs_bytes from tornado import httputil from tornado import iostream from tornado.log import gen_log +from tornado import netutil from tornado.tcpserver import TCPServer from tornado import stack_context from tornado.util import bytes_type try: - import Cookie # py2 + import Cookie # py2 except ImportError: - import http.cookies as Cookie # py3 + import http.cookies as Cookie # py3 class HTTPServer(TCPServer): @@ -54,8 +55,8 @@ class HTTPServer(TCPServer): requests). A simple example server that echoes back the URI you requested:: - import httpserver - import ioloop + import tornado.httpserver + import tornado.ioloop def handle_request(request): message = "You requested %s\n" % request.uri @@ -63,9 +64,9 @@ class HTTPServer(TCPServer): len(message), message)) request.finish() - http_server = httpserver.HTTPServer(handle_request) + http_server = tornado.httpserver.HTTPServer(handle_request) http_server.listen(8888) - ioloop.IOLoop.instance().start() + tornado.ioloop.IOLoop.instance().start() `HTTPServer` is a very basic connection handler. It parses the request headers and body, but the request callback is responsible for producing @@ -92,11 +93,10 @@ class HTTPServer(TCPServer): if Tornado is run behind an SSL-decoding proxy that does not set one of the supported ``xheaders``. - `HTTPServer` can serve SSL traffic with Python 2.6+ and OpenSSL. - To make this server serve SSL traffic, send the ssl_options dictionary + To make this server serve SSL traffic, send the ``ssl_options`` dictionary argument with the arguments required for the `ssl.wrap_socket` method, - including "certfile" and "keyfile". In Python 3.2+ you can pass - an `ssl.SSLContext` object instead of a dict:: + including ``certfile`` and ``keyfile``. (In Python 3.2+ you can pass + an `ssl.SSLContext` object instead of a dict):: HTTPServer(applicaton, ssl_options={ "certfile": os.path.join(data_dir, "mydomain.crt"), @@ -123,9 +123,9 @@ class HTTPServer(TCPServer): server.start(0) # Forks multiple sub-processes IOLoop.instance().start() - When using this interface, an `IOLoop` must *not* be passed - to the `HTTPServer` constructor. `start` will always start - the server on the default singleton `IOLoop`. + When using this interface, an `.IOLoop` must *not* be passed + to the `HTTPServer` constructor. `~.TCPServer.start` will always start + the server on the default singleton `.IOLoop`. 3. `~tornado.tcpserver.TCPServer.add_sockets`: advanced multi-process:: @@ -135,21 +135,21 @@ class HTTPServer(TCPServer): server.add_sockets(sockets) IOLoop.instance().start() - The `add_sockets` interface is more complicated, but it can be - used with `tornado.process.fork_processes` to give you more - flexibility in when the fork happens. `add_sockets` can - also be used in single-process servers if you want to create - your listening sockets in some way other than - `tornado.netutil.bind_sockets`. + The `~.TCPServer.add_sockets` interface is more complicated, + but it can be used with `tornado.process.fork_processes` to + give you more flexibility in when the fork happens. + `~.TCPServer.add_sockets` can also be used in single-process + servers if you want to create your listening sockets in some + way other than `tornado.netutil.bind_sockets`. """ - def __init__(self, request_callback, no_keep_alive=False, io_loop=None, - xheaders=False, ssl_options=None, protocol=None, **kwargs): + def __init__(self, request_callback, no_keep_alive = False, io_loop = None, + xheaders = False, ssl_options = None, protocol = None, **kwargs): self.request_callback = request_callback self.no_keep_alive = no_keep_alive self.xheaders = xheaders self.protocol = protocol - TCPServer.__init__(self, io_loop=io_loop, ssl_options=ssl_options, + TCPServer.__init__(self, io_loop = io_loop, ssl_options = ssl_options, **kwargs) def handle_stream(self, stream, address): @@ -168,8 +168,8 @@ class HTTPConnection(object): We parse HTTP headers and bodies, and execute the request callback until the HTTP conection is closed. """ - def __init__(self, stream, address, request_callback, no_keep_alive=False, - xheaders=False, protocol=None): + def __init__(self, stream, address, request_callback, no_keep_alive = False, + xheaders = False, protocol = None): self.stream = stream self.address = address # Save the socket's address family now so we know how to @@ -182,31 +182,51 @@ class HTTPConnection(object): self.protocol = protocol self._request = None self._request_finished = False + self._write_callback = None + self._close_callback = None # Save stack context here, outside of any request. This keeps # contexts from one request from leaking into the next. self._header_callback = stack_context.wrap(self._on_headers) self.stream.read_until(b"\r\n\r\n", self._header_callback) + + def _clear_callbacks(self): + """Clears the per-request callbacks. + + This is run in between requests to allow the previous handler + to be garbage collected (and prevent spurious close callbacks), + and when the connection is closed (to break up cycles and + facilitate garbage collection in cpython). + """ self._write_callback = None self._close_callback = None def set_close_callback(self, callback): + """Sets a callback that will be run when the connection is closed. + + Use this instead of accessing + `HTTPConnection.stream.set_close_callback + <.BaseIOStream.set_close_callback>` directly (which was the + recommended approach prior to Tornado 3.0). + """ self._close_callback = stack_context.wrap(callback) self.stream.set_close_callback(self._on_connection_close) def _on_connection_close(self): callback = self._close_callback self._close_callback = None - callback() + if callback: callback() # Delete any unfinished callbacks to break up reference cycles. - self._write_callback = None + self._header_callback = None + self._clear_callbacks() def close(self): self.stream.close() # Remove this reference to self, which would otherwise cause a # cycle and delay garbage collection of this connection. self._header_callback = None + self._clear_callbacks() - def write(self, chunk, callback=None): + def write(self, chunk, callback = None): """Writes a chunk of output to the stream.""" assert self._request, "Request closed" if not self.stream.closed(): @@ -251,6 +271,7 @@ class HTTPConnection(object): disconnect = True self._request = None self._request_finished = False + self._clear_callbacks() if disconnect: self.close() return @@ -273,7 +294,11 @@ class HTTPConnection(object): raise _BadRequestException("Malformed HTTP request line") if not version.startswith("HTTP/"): raise _BadRequestException("Malformed HTTP version in HTTP Request-Line") - headers = httputil.HTTPHeaders.parse(data[eol:]) + try: + headers = httputil.HTTPHeaders.parse(data[eol:]) + except ValueError: + # Probably from split() if there was no ':' in the line + raise _BadRequestException("Malformed HTTP headers") # HTTPRequest wants an IP, not a full socket address if self.address_family in (socket.AF_INET, socket.AF_INET6): @@ -283,8 +308,8 @@ class HTTPConnection(object): remote_ip = '0.0.0.0' self._request = HTTPRequest( - connection=self, method=method, uri=uri, version=version, - headers=headers, remote_ip=remote_ip, protocol=self.protocol) + connection = self, method = method, uri = uri, version = version, + headers = headers, remote_ip = remote_ip, protocol = self.protocol) content_length = headers.get("Content-Length") if content_length: @@ -339,7 +364,7 @@ class HTTPRequest(object): .. attribute:: headers - `HTTPHeader` dictionary-like object for request headers. Acts like + `.HTTPHeaders` dictionary-like object for request headers. Acts like a case-insensitive dictionary with additional methods for repeated headers. @@ -349,13 +374,13 @@ class HTTPRequest(object): .. attribute:: remote_ip - Client's IP address as a string. If `HTTPServer.xheaders` is set, + Client's IP address as a string. If ``HTTPServer.xheaders`` is set, will pass along the real IP address provided by a load balancer in the ``X-Real-Ip`` header .. attribute:: protocol - The protocol used, either "http" or "https". If `HTTPServer.xheaders` + The protocol used, either "http" or "https". If ``HTTPServer.xheaders`` is set, will pass along the protocol used by a load balancer if reported via an ``X-Scheme`` header. @@ -369,13 +394,13 @@ class HTTPRequest(object): maps arguments names to lists of values (to support multiple values for individual names). Names are of type `str`, while arguments are byte strings. Note that this is different from - `RequestHandler.get_argument`, which returns argument values as + `.RequestHandler.get_argument`, which returns argument values as unicode strings. .. attribute:: files File uploads are available in the files property, which maps file - names to lists of :class:`HTTPFile`. + names to lists of `.HTTPFile`. .. attribute:: connection @@ -384,34 +409,39 @@ class HTTPRequest(object): are typically kept open in HTTP/1.1, multiple requests can be handled sequentially on a single connection. """ - def __init__(self, method, uri, version="HTTP/1.0", headers=None, - body=None, remote_ip=None, protocol=None, host=None, - files=None, connection=None): + def __init__(self, method, uri, version = "HTTP/1.0", headers = None, + body = None, remote_ip = None, protocol = None, host = None, + files = None, connection = None): self.method = method self.uri = uri self.version = version self.headers = headers or httputil.HTTPHeaders() self.body = body or "" + + # set remote IP and protocol + self.remote_ip = remote_ip + if protocol: + self.protocol = protocol + elif connection and isinstance(connection.stream, + iostream.SSLIOStream): + self.protocol = "https" + else: + self.protocol = "http" + + # xheaders can override the defaults if connection and connection.xheaders: # Squid uses X-Forwarded-For, others use X-Real-Ip - self.remote_ip = self.headers.get( - "X-Real-Ip", self.headers.get("X-Forwarded-For", remote_ip)) - if not self._valid_ip(self.remote_ip): - self.remote_ip = remote_ip + ip = self.headers.get( + "X-Real-Ip", self.headers.get("X-Forwarded-For", self.remote_ip)) + if netutil.is_valid_ip(ip): + self.remote_ip = ip # AWS uses X-Forwarded-Proto - self.protocol = self.headers.get( - "X-Scheme", self.headers.get("X-Forwarded-Proto", protocol)) - if self.protocol not in ("http", "https"): - self.protocol = "http" - else: - self.remote_ip = remote_ip - if protocol: - self.protocol = protocol - elif connection and isinstance(connection.stream, - iostream.SSLIOStream): - self.protocol = "https" - else: - self.protocol = "http" + proto = self.headers.get( + "X-Scheme", self.headers.get("X-Forwarded-Proto", self.protocol)) + if proto in ("http", "https"): + self.protocol = proto + + self.host = host or self.headers.get("Host") or "127.0.0.1" self.files = files or {} self.connection = connection @@ -419,7 +449,7 @@ class HTTPRequest(object): self._finish_time = None self.path, sep, self.query = uri.partition('?') - self.arguments = parse_qs_bytes(self.query, keep_blank_values=True) + self.arguments = parse_qs_bytes(self.query, keep_blank_values = True) def supports_http_1_1(self): """Returns True if this request supports HTTP/1.1 semantics""" @@ -438,10 +468,10 @@ class HTTPRequest(object): self._cookies = {} return self._cookies - def write(self, chunk, callback=None): + def write(self, chunk, callback = None): """Writes the given chunk to the response stream.""" assert isinstance(chunk, bytes_type) - self.connection.write(chunk, callback=callback) + self.connection.write(chunk, callback = callback) def finish(self): """Finishes this HTTP request on the open connection.""" @@ -459,7 +489,7 @@ class HTTPRequest(object): else: return self._finish_time - self._start_time - def get_ssl_certificate(self, binary_form=False): + def get_ssl_certificate(self, binary_form = False): """Returns the client's SSL certificate, if any. To use client certificates, the HTTPServer must have been constructed @@ -481,7 +511,7 @@ class HTTPRequest(object): """ try: return self.connection.stream.socket.getpeercert( - binary_form=binary_form) + binary_form = binary_form) except ssl.SSLError: return None @@ -491,15 +521,3 @@ class HTTPRequest(object): args = ", ".join(["%s=%r" % (n, getattr(self, n)) for n in attrs]) return "%s(%s, headers=%s)" % ( self.__class__.__name__, args, dict(self.headers)) - - def _valid_ip(self, ip): - try: - res = socket.getaddrinfo(ip, 0, socket.AF_UNSPEC, - socket.SOCK_STREAM, - 0, socket.AI_NUMERICHOST) - return bool(res) - except socket.gaierror as e: - if e.args[0] == socket.EAI_NONAME: - return False - raise - return True diff --git a/libs/tornado/httputil.py b/libs/tornado/httputil.py index 94b8ba4f..a09aeabf 100755 --- a/libs/tornado/httputil.py +++ b/libs/tornado/httputil.py @@ -32,6 +32,10 @@ try: except ImportError: from http.client import responses # py3 +# responses is unused in this file, but we re-export it to other files. +# Reference it so pyflakes doesn't complain. +responses + try: from urllib import urlencode # py2 except ImportError: @@ -39,11 +43,12 @@ except ImportError: class HTTPHeaders(dict): - """A dictionary that maintains Http-Header-Case for all keys. + """A dictionary that maintains ``Http-Header-Case`` for all keys. Supports multiple values per key via a pair of new methods, - add() and get_list(). The regular dictionary interface returns a single - value per key, with multiple values joined by a comma. + `add()` and `get_list()`. The regular dictionary interface + returns a single value per key, with multiple values joined by a + comma. >>> h = HTTPHeaders({"content-type": "text/html"}) >>> list(h.keys()) @@ -209,13 +214,14 @@ def url_concat(url, args): class HTTPFile(ObjectDict): - """Represents an HTTP file. For backwards compatibility, its instance - attributes are also accessible as dictionary keys. + """Represents a file uploaded via a form. - :ivar filename: - :ivar body: - :ivar content_type: The content_type comes from the provided HTTP header - and should not be trusted outright given that it can be easily forged. + For backwards compatibility, its instance attributes are also + accessible as dictionary keys. + + * ``filename`` + * ``body`` + * ``content_type`` """ pass @@ -223,15 +229,15 @@ class HTTPFile(ObjectDict): def parse_body_arguments(content_type, body, arguments, files): """Parses a form request body. - Supports "application/x-www-form-urlencoded" and "multipart/form-data". - The content_type parameter should be a string and body should be - a byte string. The arguments and files parameters are dictionaries - that will be updated with the parsed contents. + Supports ``application/x-www-form-urlencoded`` and + ``multipart/form-data``. The ``content_type`` parameter should be + a string and ``body`` should be a byte string. The ``arguments`` + and ``files`` parameters are dictionaries that will be updated + with the parsed contents. """ if content_type.startswith("application/x-www-form-urlencoded"): - uri_arguments = parse_qs_bytes(native_str(body)) + uri_arguments = parse_qs_bytes(native_str(body), keep_blank_values=True) for name, values in uri_arguments.items(): - values = [v for v in values if v] if values: arguments.setdefault(name, []).extend(values) elif content_type.startswith("multipart/form-data"): @@ -246,9 +252,9 @@ def parse_body_arguments(content_type, body, arguments, files): def parse_multipart_form_data(boundary, data, arguments, files): - """Parses a multipart/form-data body. + """Parses a ``multipart/form-data`` body. - The boundary and data parameters are both byte strings. + The ``boundary`` and ``data`` parameters are both byte strings. The dictionaries given in the arguments and files parameters will be updated with the contents of the body. """ @@ -294,8 +300,8 @@ def parse_multipart_form_data(boundary, data, arguments, files): def format_timestamp(ts): """Formats a timestamp in the format used by HTTP. - The argument may be a numeric timestamp as returned by `time.time()`, - a time tuple as returned by `time.gmtime()`, or a `datetime.datetime` + The argument may be a numeric timestamp as returned by `time.time`, + a time tuple as returned by `time.gmtime`, or a `datetime.datetime` object. >>> format_timestamp(1359312200) @@ -314,6 +320,8 @@ def format_timestamp(ts): # _parseparam and _parse_header are copied and modified from python2.7's cgi.py # The original 2.7 version of this code did not correctly support some # combinations of semicolons and double quotes. + + def _parseparam(s): while s[:1] == ';': s = s[1:] diff --git a/libs/tornado/ioloop.py b/libs/tornado/ioloop.py index 4062661b..dd9639c0 100755 --- a/libs/tornado/ioloop.py +++ b/libs/tornado/ioloop.py @@ -36,11 +36,12 @@ import logging import numbers import os import select +import sys import threading import time import traceback -from tornado.concurrent import DummyFuture +from tornado.concurrent import Future, TracebackFuture from tornado.log import app_log, gen_log from tornado import stack_context from tornado.util import Configurable @@ -50,11 +51,6 @@ try: except ImportError: signal = None -try: - from concurrent import futures -except ImportError: - futures = None - try: import thread # py2 except ImportError: @@ -63,14 +59,18 @@ except ImportError: from tornado.platform.auto import set_close_exec, Waker +class TimeoutError(Exception): + pass + + class IOLoop(Configurable): """A level-triggered I/O loop. - We use epoll (Linux) or kqueue (BSD and Mac OS X; requires python - 2.6+) if they are available, or else we fall back on select(). If - you are implementing a system that needs to handle thousands of - simultaneous connections, you should use a system that supports either - epoll or kqueue. + We use ``epoll`` (Linux) or ``kqueue`` (BSD and Mac OS X) if they + are available, or else we fall back on select(). If you are + implementing a system that needs to handle thousands of + simultaneous connections, you should use a system that supports + either ``epoll`` or ``kqueue``. Example usage for a simple TCP server:: @@ -125,19 +125,11 @@ class IOLoop(Configurable): @staticmethod def instance(): - """Returns a global IOLoop instance. + """Returns a global `IOLoop` instance. - Most single-threaded applications have a single, global IOLoop. - Use this method instead of passing around IOLoop instances - throughout your code. - - A common pattern for classes that depend on IOLoops is to use - a default argument to enable programs with multiple IOLoops - but not require the argument for simpler applications:: - - class MyClass(object): - def __init__(self, io_loop=None): - self.io_loop = io_loop or IOLoop.instance() + Most applications have a single, global `IOLoop` running on the + main thread. Use this method to get this instance from + another thread. To get the current thread's `IOLoop`, use `current()`. """ if not hasattr(IOLoop, "_instance"): with IOLoop._instance_lock: @@ -152,27 +144,54 @@ class IOLoop(Configurable): return hasattr(IOLoop, "_instance") def install(self): - """Installs this IOloop object as the singleton instance. + """Installs this `IOLoop` object as the singleton instance. This is normally not necessary as `instance()` will create - an IOLoop on demand, but you may want to call `install` to use - a custom subclass of IOLoop. + an `IOLoop` on demand, but you may want to call `install` to use + a custom subclass of `IOLoop`. """ assert not IOLoop.initialized() IOLoop._instance = self @staticmethod def current(): + """Returns the current thread's `IOLoop`. + + If an `IOLoop` is currently running or has been marked as current + by `make_current`, returns that instance. Otherwise returns + `IOLoop.instance()`, i.e. the main thread's `IOLoop`. + + A common pattern for classes that depend on ``IOLoops`` is to use + a default argument to enable programs with multiple ``IOLoops`` + but not require the argument for simpler applications:: + + class MyClass(object): + def __init__(self, io_loop=None): + self.io_loop = io_loop or IOLoop.current() + + In general you should use `IOLoop.current` as the default when + constructing an asynchronous object, and use `IOLoop.instance` + when you mean to communicate to the main thread from a different + one. + """ current = getattr(IOLoop._current, "instance", None) if current is None: - raise ValueError("no current IOLoop") + return IOLoop.instance() return current def make_current(self): + """Makes this the `IOLoop` for the current thread. + + An `IOLoop` automatically becomes current for its thread + when it is started, but it is sometimes useful to call + `make_current` explictly before starting the `IOLoop`, + so that code run at startup time can find the right + instance. + """ IOLoop._current.instance = self - def clear_current(self): - assert IOLoop._current.instance is self + @staticmethod + def clear_current(): IOLoop._current.instance = None @classmethod @@ -195,19 +214,20 @@ class IOLoop(Configurable): pass def close(self, all_fds=False): - """Closes the IOLoop, freeing any resources used. + """Closes the `IOLoop`, freeing any resources used. If ``all_fds`` is true, all file descriptors registered on the - IOLoop will be closed (not just the ones created by the IOLoop itself). + IOLoop will be closed (not just the ones created by the + `IOLoop` itself). - Many applications will only use a single IOLoop that runs for the - entire lifetime of the process. In that case closing the IOLoop + Many applications will only use a single `IOLoop` that runs for the + entire lifetime of the process. In that case closing the `IOLoop` is not necessary since everything will be cleaned up when the process exits. `IOLoop.close` is provided mainly for scenarios such as unit tests, which create and destroy a large number of - IOLoops. + ``IOLoops``. - An IOLoop must be completely stopped before it can be closed. This + An `IOLoop` must be completely stopped before it can be closed. This means that `IOLoop.stop()` must be called *and* `IOLoop.start()` must be allowed to return before attempting to call `IOLoop.close()`. Therefore the call to `close` will usually appear just after @@ -216,7 +236,13 @@ class IOLoop(Configurable): raise NotImplementedError() def add_handler(self, fd, handler, events): - """Registers the given handler to receive the given events for fd.""" + """Registers the given handler to receive the given events for fd. + + The ``events`` argument is a bitwise or of the constants + ``IOLoop.READ``, ``IOLoop.WRITE``, and ``IOLoop.ERROR``. + + When an event occurs, ``handler(fd, events)`` will be run. + """ raise NotImplementedError() def update_handler(self, fd, events): @@ -228,28 +254,32 @@ class IOLoop(Configurable): raise NotImplementedError() def set_blocking_signal_threshold(self, seconds, action): - """Sends a signal if the ioloop is blocked for more than s seconds. + """Sends a signal if the `IOLoop` is blocked for more than + ``s`` seconds. - Pass seconds=None to disable. Requires python 2.6 on a unixy + Pass ``seconds=None`` to disable. Requires Python 2.6 on a unixy platform. - The action parameter is a python signal handler. Read the - documentation for the python 'signal' module for more information. - If action is None, the process will be killed if it is blocked for - too long. + The action parameter is a Python signal handler. Read the + documentation for the `signal` module for more information. + If ``action`` is None, the process will be killed if it is + blocked for too long. """ raise NotImplementedError() def set_blocking_log_threshold(self, seconds): - """Logs a stack trace if the ioloop is blocked for more than s seconds. - Equivalent to set_blocking_signal_threshold(seconds, self.log_stack) + """Logs a stack trace if the `IOLoop` is blocked for more than + ``s`` seconds. + + Equivalent to ``set_blocking_signal_threshold(seconds, + self.log_stack)`` """ self.set_blocking_signal_threshold(seconds, self.log_stack) def log_stack(self, signal, frame): """Signal handler to log the stack trace of the current thread. - For use with set_blocking_signal_threshold. + For use with `set_blocking_signal_threshold`. """ gen_log.warning('IOLoop blocked for %f seconds in\n%s', self._blocking_signal_threshold, @@ -258,7 +288,7 @@ class IOLoop(Configurable): def start(self): """Starts the I/O loop. - The loop will run until one of the I/O handlers calls stop(), which + The loop will run until one of the callbacks calls `stop()`, which will make the loop stop after the current event iteration completes. """ raise NotImplementedError() @@ -266,7 +296,7 @@ class IOLoop(Configurable): def stop(self): """Stop the I/O loop. - If the event loop is not currently running, the next call to start() + If the event loop is not currently running, the next call to `start()` will return immediately. To use asynchronous methods from otherwise-synchronous code (such as @@ -276,23 +306,71 @@ class IOLoop(Configurable): async_method(ioloop=ioloop, callback=ioloop.stop) ioloop.start() - ioloop.start() will return after async_method has run its callback, - whether that callback was invoked before or after ioloop.start. + ``ioloop.start()`` will return after ``async_method`` has run + its callback, whether that callback was invoked before or + after ``ioloop.start``. - Note that even after `stop` has been called, the IOLoop is not + Note that even after `stop` has been called, the `IOLoop` is not completely stopped until `IOLoop.start` has also returned. Some work that was scheduled before the call to `stop` may still - be run before the IOLoop shuts down. + be run before the `IOLoop` shuts down. """ raise NotImplementedError() + def run_sync(self, func, timeout=None): + """Starts the `IOLoop`, runs the given function, and stops the loop. + + If the function returns a `.Future`, the `IOLoop` will run + until the future is resolved. If it raises an exception, the + `IOLoop` will stop and the exception will be re-raised to the + caller. + + The keyword-only argument ``timeout`` may be used to set + a maximum duration for the function. If the timeout expires, + a `TimeoutError` is raised. + + This method is useful in conjunction with `tornado.gen.coroutine` + to allow asynchronous calls in a ``main()`` function:: + + @gen.coroutine + def main(): + # do stuff... + + if __name__ == '__main__': + IOLoop.instance().run_sync(main) + """ + future_cell = [None] + + def run(): + try: + result = func() + except Exception: + future_cell[0] = TracebackFuture() + future_cell[0].set_exc_info(sys.exc_info()) + else: + if isinstance(result, Future): + future_cell[0] = result + else: + future_cell[0] = Future() + future_cell[0].set_result(result) + self.add_future(future_cell[0], lambda future: self.stop()) + self.add_callback(run) + if timeout is not None: + timeout_handle = self.add_timeout(self.time() + timeout, self.stop) + self.start() + if timeout is not None: + self.remove_timeout(timeout_handle) + if not future_cell[0].done(): + raise TimeoutError('Operation timed out after %s seconds' % timeout) + return future_cell[0].result() + def time(self): - """Returns the current time according to the IOLoop's clock. + """Returns the current time according to the `IOLoop`'s clock. The return value is a floating-point number relative to an unspecified time in the past. - By default, the IOLoop's time function is `time.time`. However, + By default, the `IOLoop`'s time function is `time.time`. However, it may be configured to use e.g. `time.monotonic` instead. Calls to `add_timeout` that pass a number instead of a `datetime.timedelta` should use this function to compute the @@ -302,24 +380,26 @@ class IOLoop(Configurable): return time.time() def add_timeout(self, deadline, callback): - """Calls the given callback at the time deadline from the I/O loop. + """Runs the ``callback`` at the time ``deadline`` from the I/O loop. - Returns a handle that may be passed to remove_timeout to cancel. + Returns an opaque handle that may be passed to + `remove_timeout` to cancel. - ``deadline`` may be a number denoting a time relative to - `IOLoop.time`, or a ``datetime.timedelta`` object for a - deadline relative to the current time. + ``deadline`` may be a number denoting a time (on the same + scale as `IOLoop.time`, normally `time.time`), or a + `datetime.timedelta` object for a deadline relative to the + current time. Note that it is not safe to call `add_timeout` from other threads. Instead, you must use `add_callback` to transfer control to the - IOLoop's thread, and then call `add_timeout` from there. + `IOLoop`'s thread, and then call `add_timeout` from there. """ raise NotImplementedError() def remove_timeout(self, timeout): """Cancels a pending timeout. - The argument is a handle as returned by add_timeout. It is + The argument is a handle as returned by `add_timeout`. It is safe to call `remove_timeout` even if the callback has already been run. """ @@ -329,11 +409,11 @@ class IOLoop(Configurable): """Calls the given callback on the next I/O loop iteration. It is safe to call this method from any thread at any time, - except from a signal handler. Note that this is the *only* - method in IOLoop that makes this thread-safety guarantee; all - other interaction with the IOLoop must be done from that - IOLoop's thread. add_callback() may be used to transfer - control from other threads to the IOLoop's thread. + except from a signal handler. Note that this is the **only** + method in `IOLoop` that makes this thread-safety guarantee; all + other interaction with the `IOLoop` must be done from that + `IOLoop`'s thread. `add_callback()` may be used to transfer + control from other threads to the `IOLoop`'s thread. To add a callback from a signal handler, see `add_callback_from_signal`. @@ -347,22 +427,19 @@ class IOLoop(Configurable): otherwise. Callbacks added with this method will be run without any - stack_context, to avoid picking up the context of the function + `.stack_context`, to avoid picking up the context of the function that was interrupted by the signal. """ raise NotImplementedError() - if futures is not None: - _FUTURE_TYPES = (futures.Future, DummyFuture) - else: - _FUTURE_TYPES = DummyFuture - def add_future(self, future, callback): - """Schedules a callback on the IOLoop when the given future is finished. + """Schedules a callback on the ``IOLoop`` when the given + `.Future` is finished. - The callback is invoked with one argument, the future. + The callback is invoked with one argument, the + `.Future`. """ - assert isinstance(future, IOLoop._FUTURE_TYPES) + assert isinstance(future, Future) callback = stack_context.wrap(callback) future.add_done_callback( lambda future: self.add_callback(callback, future)) @@ -378,14 +455,14 @@ class IOLoop(Configurable): self.handle_callback_exception(callback) def handle_callback_exception(self, callback): - """This method is called whenever a callback run by the IOLoop + """This method is called whenever a callback run by the `IOLoop` throws an exception. By default simply logs the exception as an error. Subclasses may override this method to customize reporting of exceptions. The exception itself is not passed explicitly, but is available - in sys.exc_info. + in `sys.exc_info`. """ app_log.error("Exception in callback %r", callback, exc_info=True) @@ -428,7 +505,11 @@ class PollIOLoop(IOLoop): if all_fds: for fd in self._handlers.keys(): try: - os.close(fd) + close_method = getattr(fd, 'close', None) + if close_method is not None: + close_method() + else: + os.close(fd) except Exception: gen_log.debug("error closing fd %s", fd, exc_info=True) self._waker.close() @@ -684,16 +765,16 @@ class _Timeout(object): class PeriodicCallback(object): """Schedules the given callback to be called periodically. - The callback is called every callback_time milliseconds. + The callback is called every ``callback_time`` milliseconds. - `start` must be called after the PeriodicCallback is created. + `start` must be called after the `PeriodicCallback` is created. """ def __init__(self, callback, callback_time, io_loop=None): self.callback = callback if callback_time <= 0: raise ValueError("Periodic callback must have a positive callback_time") self.callback_time = callback_time - self.io_loop = io_loop or IOLoop.instance() + self.io_loop = io_loop or IOLoop.current() self._running = False self._timeout = None diff --git a/libs/tornado/iostream.py b/libs/tornado/iostream.py index 86cd68a8..16b0fac1 100755 --- a/libs/tornado/iostream.py +++ b/libs/tornado/iostream.py @@ -58,7 +58,7 @@ class BaseIOStream(object): All of the methods take callbacks (since writing and reading are non-blocking and asynchronous). - When a stream is closed due to an error, the IOStream's `error` + When a stream is closed due to an error, the IOStream's ``error`` attribute contains the exception object. Subclasses must implement `fileno`, `close_fd`, `write_to_fd`, @@ -66,7 +66,7 @@ class BaseIOStream(object): """ def __init__(self, io_loop=None, max_buffer_size=104857600, read_chunk_size=4096): - self.io_loop = io_loop or ioloop.IOLoop.instance() + self.io_loop = io_loop or ioloop.IOLoop.current() self.max_buffer_size = max_buffer_size self.read_chunk_size = read_chunk_size self.error = None @@ -110,16 +110,17 @@ class BaseIOStream(object): def read_from_fd(self): """Attempts to read from the underlying file. - Returns ``None`` if there was nothing to read (the socket returned - EWOULDBLOCK or equivalent), otherwise returns the data. When possible, - should return no more than ``self.read_chunk_size`` bytes at a time. + Returns ``None`` if there was nothing to read (the socket + returned `~errno.EWOULDBLOCK` or equivalent), otherwise + returns the data. When possible, should return no more than + ``self.read_chunk_size`` bytes at a time. """ raise NotImplementedError() def get_fd_error(self): """Returns information about any error on the underlying file. - This method is called after the IOLoop has signaled an error on the + This method is called after the `.IOLoop` has signaled an error on the file descriptor, and should return an Exception (such as `socket.error` with additional information, or None if no such information is available. @@ -127,23 +128,32 @@ class BaseIOStream(object): return None def read_until_regex(self, regex, callback): - """Call callback when we read the given regex pattern.""" + """Run ``callback`` when we read the given regex pattern. + + The callback will get the data read (including the data that + matched the regex and anything that came before it) as an argument. + """ self._set_read_callback(callback) self._read_regex = re.compile(regex) self._try_inline_read() def read_until(self, delimiter, callback): - """Call callback when we read the given delimiter.""" + """Run ``callback`` when we read the given delimiter. + + The callback will get the data read (including the delimiter) + as an argument. + """ self._set_read_callback(callback) self._read_delimiter = delimiter self._try_inline_read() def read_bytes(self, num_bytes, callback, streaming_callback=None): - """Call callback when we read the given number of bytes. + """Run callback when we read the given number of bytes. If a ``streaming_callback`` is given, it will be called with chunks of data as they become available, and the argument to the final - ``callback`` will be empty. + ``callback`` will be empty. Otherwise, the ``callback`` gets + the data as an argument. """ self._set_read_callback(callback) assert isinstance(num_bytes, numbers.Integral) @@ -156,7 +166,8 @@ class BaseIOStream(object): If a ``streaming_callback`` is given, it will be called with chunks of data as they become available, and the argument to the final - ``callback`` will be empty. + ``callback`` will be empty. Otherwise, the ``callback`` gets the + data as an argument. Subject to ``max_buffer_size`` limit from `IOStream` constructor if a ``streaming_callback`` is not used. @@ -174,12 +185,12 @@ class BaseIOStream(object): return self._read_until_close = True self._streaming_callback = stack_context.wrap(streaming_callback) - self._add_io_state(self.io_loop.READ) + self._try_inline_read() def write(self, data, callback=None): """Write the given data to this stream. - If callback is given, we call it when all of the buffered write + If ``callback`` is given, we call it when all of the buffered write data has been successfully written to the stream. If there was previously buffered write data and an old write callback, that callback is simply overwritten with this new callback. @@ -213,7 +224,7 @@ class BaseIOStream(object): """Close this stream. If ``exc_info`` is true, set the ``error`` attribute to the current - exception from `sys.exc_info()` (or if ``exc_info`` is a tuple, + exception from `sys.exc_info` (or if ``exc_info`` is a tuple, use that instead of `sys.exc_info`). """ if not self.closed(): @@ -573,44 +584,45 @@ class BaseIOStream(object): class IOStream(BaseIOStream): - r"""Socket-based IOStream implementation. + r"""Socket-based `IOStream` implementation. This class supports the read and write methods from `BaseIOStream` plus a `connect` method. - The socket parameter may either be connected or unconnected. For - server operations the socket is the result of calling socket.accept(). - For client operations the socket is created with socket.socket(), - and may either be connected before passing it to the IOStream or - connected with IOStream.connect. + The ``socket`` parameter may either be connected or unconnected. + For server operations the socket is the result of calling + `socket.accept `. For client operations the + socket is created with `socket.socket`, and may either be + connected before passing it to the `IOStream` or connected with + `IOStream.connect`. A very simple (and broken) HTTP client using this class:: - from tornado import ioloop - from tornado import iostream + import tornado.ioloop + import tornado.iostream import socket def send_request(): - stream.write("GET / HTTP/1.0\r\nHost: friendfeed.com\r\n\r\n") - stream.read_until("\r\n\r\n", on_headers) + stream.write(b"GET / HTTP/1.0\r\nHost: friendfeed.com\r\n\r\n") + stream.read_until(b"\r\n\r\n", on_headers) def on_headers(data): headers = {} - for line in data.split("\r\n"): - parts = line.split(":") + for line in data.split(b"\r\n"): + parts = line.split(b":") if len(parts) == 2: headers[parts[0].strip()] = parts[1].strip() - stream.read_bytes(int(headers["Content-Length"]), on_body) + stream.read_bytes(int(headers[b"Content-Length"]), on_body) def on_body(data): print data stream.close() - ioloop.IOLoop.instance().stop() + tornado.ioloop.IOLoop.instance().stop() s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0) - stream = iostream.IOStream(s) + stream = tornado.iostream.IOStream(s) stream.connect(("friendfeed.com", 80), send_request) - ioloop.IOLoop.instance().start() + tornado.ioloop.IOLoop.instance().start() """ def __init__(self, socket, *args, **kwargs): self.socket = socket @@ -650,20 +662,20 @@ class IOStream(BaseIOStream): May only be called if the socket passed to the constructor was not previously connected. The address parameter is in the - same format as for socket.connect, i.e. a (host, port) tuple. - If callback is specified, it will be called when the - connection is completed. + same format as for `socket.connect `, + i.e. a ``(host, port)`` tuple. If ``callback`` is specified, + it will be called when the connection is completed. If specified, the ``server_hostname`` parameter will be used in SSL connections for certificate validation (if requested in the ``ssl_options``) and SNI (if supported; requires Python 3.2+). - Note that it is safe to call IOStream.write while the - connection is pending, in which case the data will be written - as soon as the connection is ready. Calling IOStream read - methods before the socket is connected works on some platforms - but is non-portable. + Note that it is safe to call `IOStream.write + ` while the connection is pending, in + which case the data will be written as soon as the connection + is ready. Calling `IOStream` read methods before the socket is + connected works on some platforms but is non-portable. """ self._connecting = True try: @@ -711,13 +723,11 @@ class SSLIOStream(IOStream): ssl.wrap_socket(sock, do_handshake_on_connect=False, **kwargs) - before constructing the SSLIOStream. Unconnected sockets will be - wrapped when IOStream.connect is finished. + before constructing the `SSLIOStream`. Unconnected sockets will be + wrapped when `IOStream.connect` is finished. """ def __init__(self, *args, **kwargs): - """Creates an SSLIOStream. - - The ``ssl_options`` keyword argument may either be a dictionary + """The ``ssl_options`` keyword argument may either be a dictionary of keywords arguments for `ssl.wrap_socket`, or an `ssl.SSLContext` object. """ @@ -863,10 +873,12 @@ class SSLIOStream(IOStream): class PipeIOStream(BaseIOStream): - """Pipe-based IOStream implementation. + """Pipe-based `IOStream` implementation. The constructor takes an integer file descriptor (such as one returned - by `os.pipe`) rather than an open file object. + by `os.pipe`) rather than an open file object. Pipes are generally + one-way, so a `PipeIOStream` can be used for reading or writing but not + both. """ def __init__(self, fd, *args, **kwargs): self.fd = fd diff --git a/libs/tornado/locale.py b/libs/tornado/locale.py index e4e1a154..66e9ff6d 100755 --- a/libs/tornado/locale.py +++ b/libs/tornado/locale.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# +# -*- coding: utf-8 -*- # Copyright 2009 Facebook # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -18,25 +18,25 @@ To load a locale and generate a translated string:: - user_locale = locale.get("es_LA") + user_locale = tornado.locale.get("es_LA") print user_locale.translate("Sign out") -locale.get() returns the closest matching locale, not necessarily the +`tornado.locale.get()` returns the closest matching locale, not necessarily the specific locale you requested. You can support pluralization with -additional arguments to translate(), e.g.:: +additional arguments to `~Locale.translate()`, e.g.:: people = [...] message = user_locale.translate( "%(list)s is online", "%(list)s are online", len(people)) print message % {"list": user_locale.list(people)} -The first string is chosen if len(people) == 1, otherwise the second +The first string is chosen if ``len(people) == 1``, otherwise the second string is chosen. -Applications should call one of load_translations (which uses a simple -CSV format) or load_gettext_translations (which uses the .mo format -supported by gettext and related tools). If neither method is called, -the locale.translate method will simply return the original string. +Applications should call one of `load_translations` (which uses a simple +CSV format) or `load_gettext_translations` (which uses the ``.mo`` format +supported by `gettext` and related tools). If neither method is called, +the `Locale.translate` method will simply return the original string. """ from __future__ import absolute_import, division, print_function, with_statement @@ -63,15 +63,15 @@ def get(*locale_codes): or a loose match for the code (e.g., "en" for "en_US"), we return the locale. Otherwise we move to the next code in the list. - By default we return en_US if no translations are found for any of + By default we return ``en_US`` if no translations are found for any of the specified locales. You can change the default locale with - set_default_locale() below. + `set_default_locale()`. """ return Locale.get_closest(*locale_codes) def set_default_locale(code): - """Sets the default locale, used in get_closest_locale(). + """Sets the default locale. The default locale is assumed to be the language used for all strings in the system. The translations loaded from disk are mappings from @@ -85,32 +85,32 @@ def set_default_locale(code): def load_translations(directory): - u("""Loads translations from CSV files in a directory. + """Loads translations from CSV files in a directory. Translations are strings with optional Python-style named placeholders - (e.g., "My name is %(name)s") and their associated translations. + (e.g., ``My name is %(name)s``) and their associated translations. - The directory should have translation files of the form LOCALE.csv, - e.g. es_GT.csv. The CSV files should have two or three columns: string, + The directory should have translation files of the form ``LOCALE.csv``, + e.g. ``es_GT.csv``. The CSV files should have two or three columns: string, translation, and an optional plural indicator. Plural indicators should be one of "plural" or "singular". A given string can have both singular - and plural forms. For example "%(name)s liked this" may have a + and plural forms. For example ``%(name)s liked this`` may have a different verb conjugation depending on whether %(name)s is one name or a list of names. There should be two rows in the CSV file for that string, one with plural indicator "singular", and one "plural". For strings with no verbs that would change on translation, simply use "unknown" or the empty string (or don't include the column at all). - The file is read using the csv module in the default "excel" dialect. + The file is read using the `csv` module in the default "excel" dialect. In this format there should not be spaces after the commas. - Example translation es_LA.csv: + Example translation ``es_LA.csv``:: "I love you","Te amo" - "%(name)s liked this","A %(name)s les gust\u00f3 esto","plural" - "%(name)s liked this","A %(name)s le gust\u00f3 esto","singular" + "%(name)s liked this","A %(name)s les gustó esto","plural" + "%(name)s liked this","A %(name)s le gustó esto","singular" - """) + """ global _translations global _supported_locales _translations = {} @@ -151,22 +151,25 @@ def load_translations(directory): def load_gettext_translations(directory, domain): - """Loads translations from gettext's locale tree + """Loads translations from `gettext`'s locale tree - Locale tree is similar to system's /usr/share/locale, like: + Locale tree is similar to system's ``/usr/share/locale``, like:: - {directory}/{lang}/LC_MESSAGES/{domain}.mo + {directory}/{lang}/LC_MESSAGES/{domain}.mo Three steps are required to have you app translated: - 1. Generate POT translation file - xgettext --language=Python --keyword=_:1,2 -d cyclone file1.py file2.html etc + 1. Generate POT translation file:: - 2. Merge against existing POT file: - msgmerge old.po cyclone.po > new.po + xgettext --language=Python --keyword=_:1,2 -d mydomain file1.py file2.html etc - 3. Compile: - msgfmt cyclone.po -o {directory}/pt_BR/LC_MESSAGES/cyclone.mo + 2. Merge against existing POT file:: + + msgmerge old.po mydomain.po > new.po + + 3. Compile:: + + msgfmt mydomain.po -o {directory}/pt_BR/LC_MESSAGES/mydomain.mo """ import gettext global _translations @@ -262,9 +265,10 @@ class Locale(object): def translate(self, message, plural_message=None, count=None): """Returns the translation for the given message for this locale. - If plural_message is given, you must also provide count. We return - plural_message when count != 1, and we return the singular form - for the given message when count == 1. + If ``plural_message`` is given, you must also provide + ``count``. We return ``plural_message`` when ``count != 1``, + and we return the singular form for the given message when + ``count == 1``. """ raise NotImplementedError() @@ -273,10 +277,10 @@ class Locale(object): """Formats the given date (which should be GMT). By default, we return a relative time (e.g., "2 minutes ago"). You - can return an absolute date string with relative=False. + can return an absolute date string with ``relative=False``. You can force a full format date ("July 10, 1980") with - full_format=True. + ``full_format=True``. This method is primarily intended for dates in the past. For dates in the future, we fall back to full format. @@ -360,7 +364,7 @@ class Locale(object): """Formats the given date as a day of week. Example: "Monday, January 22". You can remove the day of week with - dow=False. + ``dow=False``. """ local_date = date - datetime.timedelta(minutes=gmt_offset) _ = self.translate @@ -421,7 +425,7 @@ class CSVLocale(Locale): class GettextLocale(Locale): - """Locale implementation using the gettext module.""" + """Locale implementation using the `gettext` module.""" def __init__(self, code, translations): try: # python 2 diff --git a/libs/tornado/netutil.py b/libs/tornado/netutil.py index 4003245e..7b7d48dd 100755 --- a/libs/tornado/netutil.py +++ b/libs/tornado/netutil.py @@ -41,14 +41,14 @@ def bind_sockets(port, address=None, family=socket.AF_UNSPEC, backlog=128, flags Address may be either an IP address or hostname. If it's a hostname, the server will listen on all IP addresses associated with the name. Address may be an empty string or None to listen on all - available interfaces. Family may be set to either socket.AF_INET - or socket.AF_INET6 to restrict to ipv4 or ipv6 addresses, otherwise + available interfaces. Family may be set to either `socket.AF_INET` + or `socket.AF_INET6` to restrict to IPv4 or IPv6 addresses, otherwise both will be used if available. The ``backlog`` argument has the same meaning as for - ``socket.listen()``. + `socket.listen() `. - ``flags`` is a bitmask of AI_* flags to ``getaddrinfo``, like + ``flags`` is a bitmask of AI_* flags to `~socket.getaddrinfo`, like ``socket.AI_PASSIVE | socket.AI_NUMERICHOST``. """ sockets = [] @@ -119,16 +119,16 @@ if hasattr(socket, 'AF_UNIX'): def add_accept_handler(sock, callback, io_loop=None): - """Adds an ``IOLoop`` event handler to accept new connections on ``sock``. + """Adds an `.IOLoop` event handler to accept new connections on ``sock``. When a connection is accepted, ``callback(connection, address)`` will be run (``connection`` is a socket object, and ``address`` is the address of the other end of the connection). Note that this signature is different from the ``callback(fd, events)`` signature used for - ``IOLoop`` handlers. + `.IOLoop` handlers. """ if io_loop is None: - io_loop = IOLoop.instance() + io_loop = IOLoop.current() def accept_handler(fd, events): while True: @@ -142,7 +142,41 @@ def add_accept_handler(sock, callback, io_loop=None): io_loop.add_handler(sock.fileno(), accept_handler, IOLoop.READ) +def is_valid_ip(ip): + """Returns true if the given string is a well-formed IP address. + + Supports IPv4 and IPv6. + """ + try: + res = socket.getaddrinfo(ip, 0, socket.AF_UNSPEC, + socket.SOCK_STREAM, + 0, socket.AI_NUMERICHOST) + return bool(res) + except socket.gaierror as e: + if e.args[0] == socket.EAI_NONAME: + return False + raise + return True + + class Resolver(Configurable): + """Configurable asynchronous DNS resolver interface. + + By default, a blocking implementation is used (which simply calls + `socket.getaddrinfo`). An alternative implementation can be + chosen with the `Resolver.configure <.Configurable.configure>` + class method:: + + Resolver.configure('tornado.netutil.ThreadedResolver') + + The implementations of this interface included with Tornado are + + * `tornado.netutil.BlockingResolver` + * `tornado.netutil.ThreadedResolver` + * `tornado.netutil.OverrideResolver` + * `tornado.platform.twisted.TwistedResolver` + * `tornado.platform.caresresolver.CaresResolver` + """ @classmethod def configurable_base(cls): return Resolver @@ -151,40 +185,64 @@ class Resolver(Configurable): def configurable_default(cls): return BlockingResolver - def getaddrinfo(self, *args, **kwargs): + def resolve(self, host, port, family=socket.AF_UNSPEC, callback=None): """Resolves an address. - The arguments to this function are the same as to - `socket.getaddrinfo`, with the addition of an optional - keyword-only ``callback`` argument. + The ``host`` argument is a string which may be a hostname or a + literal IP address. - Returns a `Future` whose result is the same as the return - value of `socket.getaddrinfo`. If a callback is passed, - it will be run with the `Future` as an argument when it - is complete. + Returns a `.Future` whose result is a list of (family, + address) pairs, where address is a tuple suitable to pass to + `socket.connect ` (i.e. a ``(host, + port)`` pair for IPv4; additional fields may be present for + IPv6). If a ``callback`` is passed, it will be run with the + result as an argument when it is complete. """ raise NotImplementedError() class ExecutorResolver(Resolver): def initialize(self, io_loop=None, executor=None): - self.io_loop = io_loop or IOLoop.instance() + self.io_loop = io_loop or IOLoop.current() self.executor = executor or dummy_executor @run_on_executor - def getaddrinfo(self, *args, **kwargs): - return socket.getaddrinfo(*args, **kwargs) + def resolve(self, host, port, family=socket.AF_UNSPEC): + addrinfo = socket.getaddrinfo(host, port, family) + results = [] + for family, socktype, proto, canonname, address in addrinfo: + results.append((family, address)) + return results + class BlockingResolver(ExecutorResolver): + """Default `Resolver` implementation, using `socket.getaddrinfo`. + + The `.IOLoop` will be blocked during the resolution, although the + callback will not be run until the next `.IOLoop` iteration. + """ def initialize(self, io_loop=None): super(BlockingResolver, self).initialize(io_loop=io_loop) + class ThreadedResolver(ExecutorResolver): + """Multithreaded non-blocking `Resolver` implementation. + + Requires the `concurrent.futures` package to be installed + (available in the standard library since Python 3.2, + installable with ``pip install futures`` in older versions). + + The thread pool size can be configured with:: + + Resolver.configure('tornado.netutil.ThreadedResolver', + num_threads=10) + """ def initialize(self, io_loop=None, num_threads=10): from concurrent.futures import ThreadPoolExecutor super(ThreadedResolver, self).initialize( io_loop=io_loop, executor=ThreadPoolExecutor(num_threads)) + class OverrideResolver(Resolver): """Wraps a resolver with a mapping of overrides. @@ -197,13 +255,12 @@ class OverrideResolver(Resolver): self.resolver = resolver self.mapping = mapping - def getaddrinfo(self, host, port, *args, **kwargs): + def resolve(self, host, port, *args, **kwargs): if (host, port) in self.mapping: host, port = self.mapping[(host, port)] elif host in self.mapping: host = self.mapping[host] - return self.resolver.getaddrinfo(host, port, *args, **kwargs) - + return self.resolver.resolve(host, port, *args, **kwargs) # These are the keyword arguments to ssl.wrap_socket that must be translated @@ -212,20 +269,22 @@ class OverrideResolver(Resolver): _SSL_CONTEXT_KEYWORDS = frozenset(['ssl_version', 'certfile', 'keyfile', 'cert_reqs', 'ca_certs', 'ciphers']) + def ssl_options_to_context(ssl_options): - """Try to Convert an ssl_options dictionary to an SSLContext object. + """Try to convert an ``ssl_options`` dictionary to an + `~ssl.SSLContext` object. The ``ssl_options`` dictionary contains keywords to be passed to - `ssl.wrap_sockets`. In Python 3.2+, `ssl.SSLContext` objects can + `ssl.wrap_socket`. In Python 3.2+, `ssl.SSLContext` objects can be used instead. This function converts the dict form to its - `SSLContext` equivalent, and may be used when a component which - accepts both forms needs to upgrade to the `SSLContext` version + `~ssl.SSLContext` equivalent, and may be used when a component which + accepts both forms needs to upgrade to the `~ssl.SSLContext` version to use features like SNI or NPN. """ if isinstance(ssl_options, dict): assert all(k in _SSL_CONTEXT_KEYWORDS for k in ssl_options), ssl_options if (not hasattr(ssl, 'SSLContext') or - isinstance(ssl_options, ssl.SSLContext)): + isinstance(ssl_options, ssl.SSLContext)): return ssl_options context = ssl.SSLContext( ssl_options.get('ssl_version', ssl.PROTOCOL_SSLv23)) @@ -241,12 +300,12 @@ def ssl_options_to_context(ssl_options): def ssl_wrap_socket(socket, ssl_options, server_hostname=None, **kwargs): - """Returns an `ssl.SSLSocket` wrapping the given socket. + """Returns an ``ssl.SSLSocket`` wrapping the given socket. ``ssl_options`` may be either a dictionary (as accepted by - `ssl_options_to_context) or an `ssl.SSLContext` object. - Additional keyword arguments are passed to `wrap_socket` - (either the `SSLContext` method or the `ssl` module function + `ssl_options_to_context`) or an `ssl.SSLContext` object. + Additional keyword arguments are passed to ``wrap_socket`` + (either the `~ssl.SSLContext` method or the `ssl` module function as appropriate). """ context = ssl_options_to_context(ssl_options) @@ -262,7 +321,7 @@ def ssl_wrap_socket(socket, ssl_options, server_hostname=None, **kwargs): else: return ssl.wrap_socket(socket, **dict(context, **kwargs)) -if hasattr(ssl, 'match_hostname'): # python 3.2+ +if hasattr(ssl, 'match_hostname') and hasattr(ssl, 'CertificateError'): # python 3.2+ ssl_match_hostname = ssl.match_hostname SSLCertificateError = ssl.CertificateError else: @@ -272,7 +331,6 @@ else: class SSLCertificateError(ValueError): pass - def _dnsname_to_pat(dn): pats = [] for frag in dn.split(r'.'): @@ -286,7 +344,6 @@ else: pats.append(frag.replace(r'\*', '[^.]*')) return re.compile(r'\A' + r'\.'.join(pats) + r'\Z', re.IGNORECASE) - def ssl_match_hostname(cert, hostname): """Verify that *cert* (in decoded format as returned by SSLSocket.getpeercert()) matches the *hostname*. RFC 2818 rules diff --git a/libs/tornado/options.py b/libs/tornado/options.py index ee146fca..b96f815d 100755 --- a/libs/tornado/options.py +++ b/libs/tornado/options.py @@ -29,27 +29,28 @@ option namespace, e.g.:: db = database.Connection(options.mysql_host) ... -The main() method of your application does not need to be aware of all of +The ``main()`` method of your application does not need to be aware of all of the options used throughout your program; they are all automatically loaded when the modules are loaded. However, all modules that define options must have been imported before the command line is parsed. -Your main() method can parse the command line or parse a config file with +Your ``main()`` method can parse the command line or parse a config file with either:: tornado.options.parse_command_line() # or tornado.options.parse_config_file("/etc/server.conf") -Command line formats are what you would expect ("--myoption=myvalue"). +Command line formats are what you would expect (``--myoption=myvalue``). Config files are just Python files. Global names become options, e.g.:: myoption = "myvalue" myotheroption = "myothervalue" -We support datetimes, timedeltas, ints, and floats (just pass a 'type' -kwarg to define). We also accept multi-value options. See the documentation -for define() below. +We support `datetimes `, `timedeltas +`, ints, and floats (just pass a ``type`` kwarg to +`define`). We also accept multi-value options. See the documentation for +`define()` below. `tornado.options.options` is a singleton instance of `OptionParser`, and the top-level functions in this module (`define`, `parse_command_line`, etc) @@ -68,7 +69,7 @@ import textwrap from tornado.escape import _unicode from tornado.log import define_logging_options from tornado import stack_context -from tornado.util import basestring_type +from tornado.util import basestring_type, exec_in class Error(Exception): @@ -103,28 +104,29 @@ class OptionParser(object): multiple=False, group=None, callback=None): """Defines a new command line option. - If type is given (one of str, float, int, datetime, or timedelta) - or can be inferred from the default, we parse the command line - arguments based on the given type. If multiple is True, we accept + If ``type`` is given (one of str, float, int, datetime, or timedelta) + or can be inferred from the ``default``, we parse the command line + arguments based on the given type. If ``multiple`` is True, we accept comma-separated values, and the option value is always a list. - For multi-value integers, we also accept the syntax x:y, which - turns into range(x, y) - very useful for long integer ranges. + For multi-value integers, we also accept the syntax ``x:y``, which + turns into ``range(x, y)`` - very useful for long integer ranges. - help and metavar are used to construct the automatically generated - command line help string. The help message is formatted like:: + ``help`` and ``metavar`` are used to construct the + automatically generated command line help string. The help + message is formatted like:: --name=METAVAR help string - group is used to group the defined options in logical + ``group`` is used to group the defined options in logical groups. By default, command line options are grouped by the file in which they are defined. Command line option names must be unique globally. They can be parsed - from the command line with parse_command_line() or parsed from a - config file with parse_config_file. + from the command line with `parse_command_line` or parsed from a + config file with `parse_config_file`. - If a callback is given, it will be run with the new value whenever + If a ``callback`` is given, it will be run with the new value whenever the option is changed. This can be used to combine command-line and file-based options:: @@ -159,9 +161,11 @@ class OptionParser(object): callback=callback) def parse_command_line(self, args=None, final=True): - """Parses all options given on the command line (defaults to sys.argv). + """Parses all options given on the command line (defaults to + `sys.argv`). - Note that args[0] is ignored since it is the program name in sys.argv. + Note that ``args[0]`` is ignored since it is the program name + in `sys.argv`. We return a list of all arguments that are not parsed as options. @@ -207,7 +211,8 @@ class OptionParser(object): from multiple sources. """ config = {} - execfile(path, config, config) + with open(path) as f: + exec_in(f.read(), config, config) for name in config: if name in self._options: self._options[name].set(config[name]) @@ -258,15 +263,16 @@ class OptionParser(object): callback() def mockable(self): - """Returns a wrapper around self that is compatible with `mock.patch`. + """Returns a wrapper around self that is compatible with + `mock.patch `. - The `mock.patch` function (included in the standard library - `unittest.mock` package since Python 3.3, or in the - third-party `mock` package for older versions of Python) is - incompatible with objects like ``options`` that override - ``__getattr__`` and ``__setattr__``. This function returns an - object that can be used with `mock.patch.object` to modify - option values:: + The `mock.patch ` function (included in + the standard library `unittest.mock` package since Python 3.3, + or in the third-party ``mock`` package for older versions of + Python) is incompatible with objects like ``options`` that + override ``__getattr__`` and ``__setattr__``. This function + returns an object that can be used with `mock.patch.object + ` to modify option values:: with mock.patch.object(options.mockable(), 'name', value): assert options.name == value diff --git a/libs/tornado/platform/caresresolver.py b/libs/tornado/platform/caresresolver.py new file mode 100755 index 00000000..7c16705d --- /dev/null +++ b/libs/tornado/platform/caresresolver.py @@ -0,0 +1,75 @@ +import pycares +import socket + +from tornado import gen +from tornado.ioloop import IOLoop +from tornado.netutil import Resolver, is_valid_ip + + +class CaresResolver(Resolver): + """Name resolver based on the c-ares library. + + This is a non-blocking and non-threaded resolver. It may not produce + the same results as the system resolver, but can be used for non-blocking + resolution when threads cannot be used. + + c-ares fails to resolve some names when ``family`` is ``AF_UNSPEC``, + so it is only recommended for use in ``AF_INET`` (i.e. IPv4). This is + the default for ``tornado.simple_httpclient``, but other libraries + may default to ``AF_UNSPEC``. + """ + def initialize(self, io_loop=None): + self.io_loop = io_loop or IOLoop.current() + self.channel = pycares.Channel(sock_state_cb=self._sock_state_cb) + self.fds = {} + + def _sock_state_cb(self, fd, readable, writable): + state = ((IOLoop.READ if readable else 0) | + (IOLoop.WRITE if writable else 0)) + if not state: + self.io_loop.remove_handler(fd) + del self.fds[fd] + elif fd in self.fds: + self.io_loop.update_handler(fd, state) + self.fds[fd] = state + else: + self.io_loop.add_handler(fd, self._handle_events, state) + self.fds[fd] = state + + def _handle_events(self, fd, events): + read_fd = pycares.ARES_SOCKET_BAD + write_fd = pycares.ARES_SOCKET_BAD + if events & IOLoop.READ: + read_fd = fd + if events & IOLoop.WRITE: + write_fd = fd + self.channel.process_fd(read_fd, write_fd) + + @gen.coroutine + def resolve(self, host, port, family=0): + if is_valid_ip(host): + addresses = [host] + else: + # gethostbyname doesn't take callback as a kwarg + self.channel.gethostbyname(host, family, (yield gen.Callback(1))) + callback_args = yield gen.Wait(1) + assert isinstance(callback_args, gen.Arguments) + assert not callback_args.kwargs + result, error = callback_args.args + if error: + raise Exception('C-Ares returned error %s: %s while resolving %s' % + (error, pycares.errno.strerror(error), host)) + addresses = result.addresses + addrinfo = [] + for address in addresses: + if '.' in address: + address_family = socket.AF_INET + elif ':' in address: + address_family = socket.AF_INET6 + else: + address_family = socket.AF_UNSPEC + if family != socket.AF_UNSPEC and family != address_family: + raise Exception('Requested socket family %d but got %d' % + (family, address_family)) + addrinfo.append((address_family, (address, port))) + raise gen.Return(addrinfo) diff --git a/libs/tornado/platform/twisted.py b/libs/tornado/platform/twisted.py index 34e108d7..910e46af 100755 --- a/libs/tornado/platform/twisted.py +++ b/libs/tornado/platform/twisted.py @@ -22,6 +22,8 @@ This module lets you run applications and libraries written for Twisted in a Tornado application. It can be used in two modes, depending on which library's underlying event loop you want to use. +This module has been tested with Twisted versions 11.0.0 and newer. + Twisted on Tornado ------------------ @@ -60,26 +62,33 @@ reactor. Recommended usage:: reactor.run() `TwistedIOLoop` always uses the global Twisted reactor. - -This module has been tested with Twisted versions 11.0.0 and newer. """ from __future__ import absolute_import, division, print_function, with_statement -import functools import datetime +import functools +import socket +import twisted.internet.abstract from twisted.internet.posixbase import PosixReactorBase from twisted.internet.interfaces import \ IReactorFDSet, IDelayedCall, IReactorTime, IReadDescriptor, IWriteDescriptor from twisted.python import failure, log from twisted.internet import error +import twisted.names.cache +import twisted.names.client +import twisted.names.hosts +import twisted.names.resolve from zope.interface import implementer -import tornado +from tornado.concurrent import return_future +from tornado.escape import utf8 +from tornado import gen import tornado.ioloop from tornado.log import app_log +from tornado.netutil import Resolver from tornado.stack_context import NullContext, wrap from tornado.ioloop import IOLoop @@ -140,7 +149,7 @@ class TornadoReactor(PosixReactorBase): """ def __init__(self, io_loop=None): if not io_loop: - io_loop = tornado.ioloop.IOLoop.instance() + io_loop = tornado.ioloop.IOLoop.current() self._io_loop = io_loop self._readers = {} # map of reader objects to fd self._writers = {} # map of writer objects to fd @@ -177,8 +186,12 @@ class TornadoReactor(PosixReactorBase): def callFromThread(self, f, *args, **kw): """See `twisted.internet.interfaces.IReactorThreads.callFromThread`""" assert callable(f), "%s is not callable" % f - p = functools.partial(f, *args, **kw) - self._io_loop.add_callback(p) + with NullContext(): + # This NullContext is mainly for an edge case when running + # TwistedIOLoop on top of a TornadoReactor. + # TwistedIOLoop.add_callback uses reactor.callFromThread and + # should not pick up additional StackContexts along the way. + self._io_loop.add_callback(f, *args, **kw) # We don't need the waker code from the super class, Tornado uses # its own waker. @@ -344,7 +357,7 @@ class _TestReactor(TornadoReactor): def install(io_loop=None): """Install this package as the default Twisted reactor.""" if not io_loop: - io_loop = tornado.ioloop.IOLoop.instance() + io_loop = tornado.ioloop.IOLoop.current() reactor = TornadoReactor(io_loop) from twisted.internet.main import installReactor installReactor(reactor) @@ -383,18 +396,21 @@ class _FD(object): class TwistedIOLoop(tornado.ioloop.IOLoop): """IOLoop implementation that runs on Twisted. - Uses the global Twisted reactor. It is possible to create multiple - TwistedIOLoops in the same process, but it doesn't really make sense - because they will all run in the same thread. + Uses the global Twisted reactor by default. To create multiple + `TwistedIOLoops` in the same process, you must pass a unique reactor + when constructing each one. Not compatible with `tornado.process.Subprocess.set_exit_callback` because the ``SIGCHLD`` handlers used by Tornado and Twisted conflict with each other. """ - def initialize(self): - from twisted.internet import reactor + def initialize(self, reactor=None): + if reactor is None: + import twisted.internet.reactor + reactor = twisted.internet.reactor self.reactor = reactor self.fds = {} + self.reactor.callWhenRunning(self.make_current) def close(self, all_fds=False): self.reactor.removeAll() @@ -456,13 +472,14 @@ class TwistedIOLoop(tornado.ioloop.IOLoop): if isinstance(deadline, (int, long, float)): delay = max(deadline - self.time(), 0) elif isinstance(deadline, datetime.timedelta): - delay = deadline.total_seconds() + delay = tornado.ioloop._Timeout.timedelta_to_seconds(deadline) else: raise TypeError("Unsupported deadline %r") return self.reactor.callLater(delay, self._run_callback, wrap(callback)) def remove_timeout(self, timeout): - timeout.cancel() + if timeout.active(): + timeout.cancel() def add_callback(self, callback, *args, **kwargs): self.reactor.callFromThread(self._run_callback, @@ -470,3 +487,58 @@ class TwistedIOLoop(tornado.ioloop.IOLoop): def add_callback_from_signal(self, callback, *args, **kwargs): self.add_callback(callback, *args, **kwargs) + + +class TwistedResolver(Resolver): + """Twisted-based asynchronous resolver. + + This is a non-blocking and non-threaded resolver. It is + recommended only when threads cannot be used, since it has + limitations compared to the standard ``getaddrinfo``-based + `~tornado.netutil.Resolver` and + `~tornado.netutil.ThreadedResolver`. Specifically, it returns at + most one result, and arguments other than ``host`` and ``family`` + are ignored. It may fail to resolve when ``family`` is not + ``socket.AF_UNSPEC``. + + Requires Twisted 12.1 or newer. + """ + def initialize(self, io_loop=None): + self.io_loop = io_loop or IOLoop.current() + # partial copy of twisted.names.client.createResolver, which doesn't + # allow for a reactor to be passed in. + self.reactor = tornado.platform.twisted.TornadoReactor(io_loop) + + host_resolver = twisted.names.hosts.Resolver('/etc/hosts') + cache_resolver = twisted.names.cache.CacheResolver(reactor=self.reactor) + real_resolver = twisted.names.client.Resolver('/etc/resolv.conf', + reactor=self.reactor) + self.resolver = twisted.names.resolve.ResolverChain( + [host_resolver, cache_resolver, real_resolver]) + + @gen.coroutine + def resolve(self, host, port, family=0): + # getHostByName doesn't accept IP addresses, so if the input + # looks like an IP address just return it immediately. + if twisted.internet.abstract.isIPAddress(host): + resolved = host + resolved_family = socket.AF_INET + elif twisted.internet.abstract.isIPv6Address(host): + resolved = host + resolved_family = socket.AF_INET6 + else: + deferred = self.resolver.getHostByName(utf8(host)) + resolved = yield gen.Task(deferred.addCallback) + if twisted.internet.abstract.isIPAddress(resolved): + resolved_family = socket.AF_INET + elif twisted.internet.abstract.isIPv6Address(resolved): + resolved_family = socket.AF_INET6 + else: + resolved_family = socket.AF_UNSPEC + if family != socket.AF_UNSPEC and family != resolved_family: + raise Exception('Requested socket family %d but got %d' % + (family, resolved_family)) + result = [ + (resolved_family, (resolved, port)), + ] + raise gen.Return(result) diff --git a/libs/tornado/process.py b/libs/tornado/process.py index 0509eb3a..438db66d 100755 --- a/libs/tornado/process.py +++ b/libs/tornado/process.py @@ -14,7 +14,9 @@ # License for the specific language governing permissions and limitations # under the License. -"""Utilities for working with multiple processes.""" +"""Utilities for working with multiple processes, including both forking +the server into multiple processes and managing subprocesses. +""" from __future__ import absolute_import, division, print_function, with_statement @@ -169,8 +171,8 @@ class Subprocess(object): additions: * ``stdin``, ``stdout``, and ``stderr`` may have the value - `tornado.process.Subprocess.STREAM`, which will make the corresponding - attribute of the resulting Subprocess a `PipeIOStream`. + ``tornado.process.Subprocess.STREAM``, which will make the corresponding + attribute of the resulting Subprocess a `.PipeIOStream`. * A new keyword argument ``io_loop`` may be used to pass in an IOLoop. """ STREAM = object() @@ -229,15 +231,15 @@ class Subprocess(object): def initialize(cls, io_loop=None): """Initializes the ``SIGCHILD`` handler. - The signal handler is run on an IOLoop to avoid locking issues. - Note that the IOLoop used for signal handling need not be the + The signal handler is run on an `.IOLoop` to avoid locking issues. + Note that the `.IOLoop` used for signal handling need not be the same one used by individual Subprocess objects (as long as the - IOLoops are each running in separate threads). + ``IOLoops`` are each running in separate threads). """ if cls._initialized: return if io_loop is None: - io_loop = ioloop.IOLoop.instance() + io_loop = ioloop.IOLoop.current() cls._old_sigchld = signal.signal( signal.SIGCHLD, lambda sig, frame: io_loop.add_callback_from_signal(cls._cleanup)) diff --git a/libs/tornado/simple_httpclient.py b/libs/tornado/simple_httpclient.py index f33ed242..117ce75b 100755 --- a/libs/tornado/simple_httpclient.py +++ b/libs/tornado/simple_httpclient.py @@ -2,7 +2,7 @@ from __future__ import absolute_import, division, print_function, with_statement from tornado.escape import utf8, _unicode, native_str -from tornado.httpclient import HTTPRequest, HTTPResponse, HTTPError, AsyncHTTPClient, main, _RequestProxy +from tornado.httpclient import HTTPResponse, HTTPError, AsyncHTTPClient, main, _RequestProxy from tornado.httputil import HTTPHeaders from tornado.iostream import IOStream, SSLIOStream from tornado.netutil import Resolver, OverrideResolver @@ -92,10 +92,12 @@ class SimpleAsyncHTTPClient(AsyncHTTPClient): request, callback = self.queue.popleft() key = object() self.active[key] = (request, callback) - _HTTPConnection(self.io_loop, self, request, - functools.partial(self._release_fetch, key), - callback, - self.max_buffer_size, self.resolver) + release_callback = functools.partial(self._release_fetch, key) + self._handle_request(request, release_callback, callback) + + def _handle_request(self, request, release_callback, final_callback): + _HTTPConnection(self.io_loop, self, request, release_callback, + final_callback, self.max_buffer_size, self.resolver) def _release_fetch(self, key): del self.active[key] @@ -150,13 +152,24 @@ class _HTTPConnection(object): # so restrict to ipv4 by default. af = socket.AF_INET - self.resolver.getaddrinfo( - host, port, af, socket.SOCK_STREAM, 0, 0, - callback=self._on_resolve) + self.resolver.resolve(host, port, af, callback=self._on_resolve) - def _on_resolve(self, future): - af, socktype, proto, canonname, sockaddr = future.result()[0] + def _on_resolve(self, addrinfo): + self.stream = self._create_stream(addrinfo) + timeout = min(self.request.connect_timeout, self.request.request_timeout) + if timeout: + self._timeout = self.io_loop.add_timeout( + self.start_time + timeout, + stack_context.wrap(self._on_timeout)) + self.stream.set_close_callback(self._on_close) + # ipv6 addresses are broken (in self.parsed.hostname) until + # 2.7, here is correctly parsed value calculated in __init__ + sockaddr = addrinfo[0][1] + self.stream.connect(sockaddr, self._on_connect, + server_hostname=self.parsed_hostname) + def _create_stream(self, addrinfo): + af = addrinfo[0][0] if self.parsed.scheme == "https": ssl_options = {} if self.request.validate_cert: @@ -189,24 +202,14 @@ class _HTTPConnection(object): # information. ssl_options["ssl_version"] = ssl.PROTOCOL_SSLv3 - self.stream = SSLIOStream(socket.socket(af, socktype, proto), - io_loop=self.io_loop, - ssl_options=ssl_options, - max_buffer_size=self.max_buffer_size) + return SSLIOStream(socket.socket(af), + io_loop=self.io_loop, + ssl_options=ssl_options, + max_buffer_size=self.max_buffer_size) else: - self.stream = IOStream(socket.socket(af, socktype, proto), - io_loop=self.io_loop, - max_buffer_size=self.max_buffer_size) - timeout = min(self.request.connect_timeout, self.request.request_timeout) - if timeout: - self._timeout = self.io_loop.add_timeout( - self.start_time + timeout, - stack_context.wrap(self._on_timeout)) - self.stream.set_close_callback(self._on_close) - # ipv6 addresses are broken (in self.parsed.hostname) until - # 2.7, here is correctly parsed value calculated in __init__ - self.stream.connect(sockaddr, self._on_connect, - server_hostname=self.parsed_hostname) + return IOStream(socket.socket(af), + io_loop=self.io_loop, + max_buffer_size=self.max_buffer_size) def _on_timeout(self): self._timeout = None @@ -246,6 +249,9 @@ class _HTTPConnection(object): username = self.request.auth_username password = self.request.auth_password or '' if username is not None: + if self.request.auth_mode not in (None, "basic"): + raise ValueError("unsupported auth_mode %s", + self.request.auth_mode) auth = utf8(username) + b":" + utf8(password) self.request.headers["Authorization"] = (b"Basic " + base64.b64encode(auth)) @@ -414,7 +420,7 @@ class _HTTPConnection(object): self.final_callback = None self._release() self.client.fetch(new_request, final_callback) - self.stream.close() + self._on_end_request() return if self._decompressor: data = (self._decompressor.decompress(data) + @@ -434,6 +440,9 @@ class _HTTPConnection(object): buffer=buffer, effective_url=self.request.url) self._run_callback(response) + self._on_end_request() + + def _on_end_request(self): self.stream.close() def _on_chunk_length(self, data): diff --git a/libs/tornado/stack_context.py b/libs/tornado/stack_context.py index c30a2598..8804d42d 100755 --- a/libs/tornado/stack_context.py +++ b/libs/tornado/stack_context.py @@ -14,20 +14,21 @@ # License for the specific language governing permissions and limitations # under the License. -"""StackContext allows applications to maintain threadlocal-like state +"""`StackContext` allows applications to maintain threadlocal-like state that follows execution as it moves to other execution contexts. The motivating examples are to eliminate the need for explicit -async_callback wrappers (as in tornado.web.RequestHandler), and to +``async_callback`` wrappers (as in `tornado.web.RequestHandler`), and to allow some additional context to be kept for logging. -This is slightly magic, but it's an extension of the idea that an exception -handler is a kind of stack-local state and when that stack is suspended -and resumed in a new context that state needs to be preserved. StackContext -shifts the burden of restoring that state from each call site (e.g. -wrapping each AsyncHTTPClient callback in async_callback) to the mechanisms -that transfer control from one context to another (e.g. AsyncHTTPClient -itself, IOLoop, thread pools, etc). +This is slightly magic, but it's an extension of the idea that an +exception handler is a kind of stack-local state and when that stack +is suspended and resumed in a new context that state needs to be +preserved. `StackContext` shifts the burden of restoring that state +from each call site (e.g. wrapping each `.AsyncHTTPClient` callback +in ``async_callback``) to the mechanisms that transfer control from +one context to another (e.g. `.AsyncHTTPClient` itself, `.IOLoop`, +thread pools, etc). Example usage:: @@ -52,7 +53,7 @@ Here are a few rules of thumb for when it's necessary: * If you're writing an asynchronous library that doesn't rely on a stack_context-aware library like `tornado.ioloop` or `tornado.iostream` (for example, if you're writing a thread pool), use - `stack_context.wrap()` before any asynchronous operations to capture the + `.stack_context.wrap()` before any asynchronous operations to capture the stack context from where the operation was started. * If you're writing an asynchronous library that has some shared @@ -68,9 +69,6 @@ Here are a few rules of thumb for when it's necessary: from __future__ import absolute_import, division, print_function, with_statement -import contextlib -import functools -import operator import sys import threading @@ -83,7 +81,7 @@ class StackContextInconsistentError(Exception): class _State(threading.local): def __init__(self): - self.contexts = () + self.contexts = (tuple(), None) _state = _State() @@ -107,34 +105,41 @@ class StackContext(object): context that are currently pending). This is an advanced feature and not necessary in most applications. """ - def __init__(self, context_factory, _active_cell=None): + def __init__(self, context_factory): self.context_factory = context_factory - self.active_cell = _active_cell or [True] + self.contexts = [] + + # StackContext protocol + def enter(self): + context = self.context_factory() + self.contexts.append(context) + context.__enter__() + + def exit(self, type, value, traceback): + context = self.contexts.pop() + context.__exit__(type, value, traceback) # Note that some of this code is duplicated in ExceptionStackContext # below. ExceptionStackContext is more common and doesn't need # the full generality of this class. def __enter__(self): self.old_contexts = _state.contexts - # _state.contexts is a tuple of (class, arg, active_cell) tuples - self.new_contexts = (self.old_contexts + - ((StackContext, self.context_factory, - self.active_cell),)) + self.new_contexts = (self.old_contexts[0] + (self,), self) _state.contexts = self.new_contexts + try: - self.context = self.context_factory() - self.context.__enter__() - except Exception: + self.enter() + except: _state.contexts = self.old_contexts raise - return lambda: operator.setitem(self.active_cell, 0, False) def __exit__(self, type, value, traceback): try: - return self.context.__exit__(type, value, traceback) + self.exit(type, value, traceback) finally: final_contexts = _state.contexts _state.contexts = self.old_contexts + # Generator coroutines and with-statements with non-local # effects interact badly. Check here for signs of # the stack getting out of sync. @@ -145,33 +150,32 @@ class StackContext(object): raise StackContextInconsistentError( 'stack_context inconsistency (may be caused by yield ' 'within a "with StackContext" block)') - self.old_contexts = self.new_contexts = None class ExceptionStackContext(object): """Specialization of StackContext for exception handling. - The supplied exception_handler function will be called in the + The supplied ``exception_handler`` function will be called in the event of an uncaught exception in this context. The semantics are similar to a try/finally clause, and intended use cases are to log an error, close a socket, or similar cleanup actions. The - exc_info triple (type, value, traceback) will be passed to the + ``exc_info`` triple ``(type, value, traceback)`` will be passed to the exception_handler function. If the exception handler returns true, the exception will be consumed and will not be propagated to other exception handlers. """ - def __init__(self, exception_handler, _active_cell=None): + def __init__(self, exception_handler): self.exception_handler = exception_handler - self.active_cell = _active_cell or [True] + + def exit(self, type, value, traceback): + if type is not None: + return self.exception_handler(type, value, traceback) def __enter__(self): self.old_contexts = _state.contexts - self.new_contexts = (self.old_contexts + - ((ExceptionStackContext, self.exception_handler, - self.active_cell),)) + self.new_contexts = (self.old_contexts[0], self) _state.contexts = self.new_contexts - return lambda: operator.setitem(self.active_cell, 0, False) def __exit__(self, type, value, traceback): try: @@ -180,98 +184,116 @@ class ExceptionStackContext(object): finally: final_contexts = _state.contexts _state.contexts = self.old_contexts + if final_contexts is not self.new_contexts: raise StackContextInconsistentError( 'stack_context inconsistency (may be caused by yield ' 'within a "with StackContext" block)') - self.old_contexts = self.new_contexts = None class NullContext(object): - """Resets the StackContext. + """Resets the `StackContext`. - Useful when creating a shared resource on demand (e.g. an AsyncHTTPClient) - where the stack that caused the creating is not relevant to future - operations. + Useful when creating a shared resource on demand (e.g. an + `.AsyncHTTPClient`) where the stack that caused the creating is + not relevant to future operations. """ def __enter__(self): self.old_contexts = _state.contexts - _state.contexts = () + _state.contexts = (tuple(), None) def __exit__(self, type, value, traceback): _state.contexts = self.old_contexts -class _StackContextWrapper(functools.partial): - pass - - def wrap(fn): - """Returns a callable object that will restore the current StackContext + """Returns a callable object that will restore the current `StackContext` when executed. Use this whenever saving a callback to be executed later in a different execution context (either in a different thread or asynchronously in the same thread). """ - if fn is None or fn.__class__ is _StackContextWrapper: + # Check if function is already wrapped + if fn is None or hasattr(fn, '_wrapped'): return fn - # functools.wraps doesn't appear to work on functools.partial objects - #@functools.wraps(fn) + # Capture current stack head + contexts = _state.contexts + + #@functools.wraps def wrapped(*args, **kwargs): - callback, contexts, args = args[0], args[1], args[2:] + try: + # Force local state - switch to new stack chain + current_state = _state.contexts + _state.contexts = contexts - if _state.contexts: - new_contexts = [NullContext()] - else: - new_contexts = [] - if contexts: - new_contexts.extend(cls(arg, active_cell) - for (cls, arg, active_cell) in contexts - if active_cell[0]) - if len(new_contexts) > 1: - with _nested(*new_contexts): - callback(*args, **kwargs) - elif new_contexts: - with new_contexts[0]: - callback(*args, **kwargs) - else: - callback(*args, **kwargs) - return _StackContextWrapper(wrapped, fn, _state.contexts) + # Current exception + exc = (None, None, None) + top = None + + # Apply stack contexts + last_ctx = 0 + stack = contexts[0] + + # Apply state + for n in stack: + try: + n.enter() + last_ctx += 1 + except: + # Exception happened. Record exception info and store top-most handler + exc = sys.exc_info() + top = n.old_contexts[1] + + # Execute callback if no exception happened while restoring state + if top is None: + try: + fn(*args, **kwargs) + except: + exc = sys.exc_info() + top = contexts[1] + + # If there was exception, try to handle it by going through the exception chain + if top is not None: + exc = _handle_exception(top, exc) + else: + # Otherwise take shorter path and run stack contexts in reverse order + while last_ctx > 0: + last_ctx -= 1 + c = stack[last_ctx] + + try: + c.exit(*exc) + except: + exc = sys.exc_info() + top = c.old_contexts[1] + break + else: + top = None + + # If if exception happened while unrolling, take longer exception handler path + if top is not None: + exc = _handle_exception(top, exc) + + # If exception was not handled, raise it + if exc != (None, None, None): + raise_exc_info(exc) + finally: + _state.contexts = current_state + + wrapped._wrapped = True + return wrapped -@contextlib.contextmanager -def _nested(*managers): - """Support multiple context managers in a single with-statement. +def _handle_exception(tail, exc): + while tail is not None: + try: + if tail.exit(*exc): + exc = (None, None, None) + except: + exc = sys.exc_info() - Copied from the python 2.6 standard library. It's no longer present - in python 3 because the with statement natively supports multiple - context managers, but that doesn't help if the list of context - managers is not known until runtime. - """ - exits = [] - vars = [] - exc = (None, None, None) - try: - for mgr in managers: - exit = mgr.__exit__ - enter = mgr.__enter__ - vars.append(enter()) - exits.append(exit) - yield vars - except: - exc = sys.exc_info() - finally: - while exits: - exit = exits.pop() - try: - if exit(*exc): - exc = (None, None, None) - except: - exc = sys.exc_info() - if exc != (None, None, None): - # Don't rely on sys.exc_info() still containing - # the right information. Another exception may - # have been raised and caught by an exit method - raise_exc_info(exc) + tail = tail.old_contexts[1] + + return exc diff --git a/libs/tornado/tcpserver.py b/libs/tornado/tcpserver.py index 52ed70b1..fbd9c63d 100755 --- a/libs/tornado/tcpserver.py +++ b/libs/tornado/tcpserver.py @@ -28,13 +28,13 @@ from tornado.iostream import IOStream, SSLIOStream from tornado.netutil import bind_sockets, add_accept_handler, ssl_wrap_socket from tornado import process + class TCPServer(object): r"""A non-blocking, single-threaded TCP server. To use `TCPServer`, define a subclass which overrides the `handle_stream` method. - `TCPServer` can serve SSL traffic with Python 2.6+ and OpenSSL. To make this server serve SSL traffic, send the ssl_options dictionary argument with the arguments required for the `ssl.wrap_socket` method, including "certfile" and "keyfile":: @@ -59,9 +59,9 @@ class TCPServer(object): server.start(0) # Forks multiple sub-processes IOLoop.instance().start() - When using this interface, an `IOLoop` must *not* be passed + When using this interface, an `.IOLoop` must *not* be passed to the `TCPServer` constructor. `start` will always start - the server on the default singleton `IOLoop`. + the server on the default singleton `.IOLoop`. 3. `add_sockets`: advanced multi-process:: @@ -76,7 +76,7 @@ class TCPServer(object): flexibility in when the fork happens. `add_sockets` can also be used in single-process servers if you want to create your listening sockets in some way other than - `bind_sockets`. + `~tornado.netutil.bind_sockets`. """ def __init__(self, io_loop=None, ssl_options=None): self.io_loop = io_loop @@ -108,7 +108,7 @@ class TCPServer(object): This method may be called more than once to listen on multiple ports. `listen` takes effect immediately; it is not necessary to call `TCPServer.start` afterwards. It is, however, necessary to start - the `IOLoop`. + the `.IOLoop`. """ sockets = bind_sockets(port, address=address) self.add_sockets(sockets) @@ -117,13 +117,13 @@ class TCPServer(object): """Makes this server start accepting connections on the given sockets. The ``sockets`` parameter is a list of socket objects such as - those returned by `bind_sockets`. + those returned by `~tornado.netutil.bind_sockets`. `add_sockets` is typically used in combination with that method and `tornado.process.fork_processes` to provide greater control over the initialization of a multi-process server. """ if self.io_loop is None: - self.io_loop = IOLoop.instance() + self.io_loop = IOLoop.current() for sock in sockets: self._sockets[sock.fileno()] = sock @@ -144,12 +144,12 @@ class TCPServer(object): Address may be either an IP address or hostname. If it's a hostname, the server will listen on all IP addresses associated with the name. Address may be an empty string or None to listen on all - available interfaces. Family may be set to either ``socket.AF_INET`` - or ``socket.AF_INET6`` to restrict to ipv4 or ipv6 addresses, otherwise + available interfaces. Family may be set to either `socket.AF_INET` + or `socket.AF_INET6` to restrict to IPv4 or IPv6 addresses, otherwise both will be used if available. The ``backlog`` argument has the same meaning as for - `socket.listen`. + `socket.listen `. This method may be called multiple times prior to `start` to listen on multiple ports or interfaces. @@ -162,7 +162,7 @@ class TCPServer(object): self._pending_sockets.extend(sockets) def start(self, num_processes=1): - """Starts this server in the IOLoop. + """Starts this server in the `.IOLoop`. By default, we run the server in this process and do not fork any additional child process. @@ -199,7 +199,7 @@ class TCPServer(object): sock.close() def handle_stream(self, stream, address): - """Override to handle a new `IOStream` from an incoming connection.""" + """Override to handle a new `.IOStream` from an incoming connection.""" raise NotImplementedError() def _handle_connection(self, connection, address): diff --git a/libs/tornado/template.py b/libs/tornado/template.py index 4f61de3f..8e1bfbae 100755 --- a/libs/tornado/template.py +++ b/libs/tornado/template.py @@ -79,9 +79,10 @@ We provide the functions escape(), url_escape(), json_encode(), and squeeze() to all templates by default. Typical applications do not create `Template` or `Loader` instances by -hand, but instead use the `render` and `render_string` methods of +hand, but instead use the `~.RequestHandler.render` and +`~.RequestHandler.render_string` methods of `tornado.web.RequestHandler`, which load templates automatically based -on the ``template_path`` `Application` setting. +on the ``template_path`` `.Application` setting. Syntax Reference ---------------- @@ -109,7 +110,7 @@ with ``{# ... #}``. ``{% autoescape *function* %}`` Sets the autoescape mode for the current file. This does not affect other files, even those referenced by ``{% include %}``. Note that - autoescaping can also be configured globally, at the `Application` + autoescaping can also be configured globally, at the `.Application` or `Loader`.:: {% autoescape xhtml_escape %} @@ -298,14 +299,14 @@ class Template(object): class BaseLoader(object): - """Base class for template loaders.""" + """Base class for template loaders. + + You must use a template loader to use template constructs like + ``{% extends %}`` and ``{% include %}``. The loader caches all + templates after they are loaded the first time. + """ def __init__(self, autoescape=_DEFAULT_AUTOESCAPE, namespace=None): - """Creates a template loader. - - root_directory may be the empty string if this loader does not - use the filesystem. - - autoescape must be either None or a string naming a function + """``autoescape`` must be either None or a string naming a function in the template namespace, such as "xhtml_escape". """ self.autoescape = autoescape @@ -341,10 +342,6 @@ class BaseLoader(object): class Loader(BaseLoader): """A template loader that loads from a single root directory. - - You must use a template loader to use template constructs like - {% extends %} and {% include %}. Loader caches all templates after - they are loaded the first time. """ def __init__(self, root_directory, **kwargs): super(Loader, self).__init__(**kwargs) diff --git a/libs/tornado/testing.py b/libs/tornado/testing.py index 30fe4e29..51663a4a 100755 --- a/libs/tornado/testing.py +++ b/libs/tornado/testing.py @@ -1,21 +1,13 @@ #!/usr/bin/env python """Support classes for automated testing. -This module contains three parts: +* `AsyncTestCase` and `AsyncHTTPTestCase`: Subclasses of unittest.TestCase + with additional support for testing asynchronous (`.IOLoop` based) code. -* `AsyncTestCase`/`AsyncHTTPTestCase`: Subclasses of unittest.TestCase - with additional support for testing asynchronous (IOLoop-based) code. - -* `LogTrapTestCase`: Subclass of unittest.TestCase that discards log output - from tests that pass and only produces output for failing tests. +* `ExpectLog` and `LogTrapTestCase`: Make test logs less spammy. * `main()`: A simple test runner (wrapper around unittest.main()) with support for the tornado.autoreload module to rerun the tests when code changes. - -These components may be used together or independently. In particular, -it is safe to combine AsyncTestCase and LogTrapTestCase via multiple -inheritance. See the docstrings for each class/function below for more -information. """ from __future__ import absolute_import, division, print_function, with_statement @@ -46,12 +38,11 @@ import re import signal import socket import sys -import types try: - from io import StringIO # py3 -except ImportError: from cStringIO import StringIO # py2 +except ImportError: + from io import StringIO # py3 # Tornado's own test suite requires the updated unittest module # (either py27+ or unittest2) so tornado.test.util enforces @@ -91,30 +82,53 @@ def bind_unused_port(): return sock, port +def get_async_test_timeout(): + """Get the global timeout setting for async tests. + + Returns a float, the timeout in seconds. + """ + try: + return float(os.environ.get('ASYNC_TEST_TIMEOUT')) + except (ValueError, TypeError): + return 5 + + class AsyncTestCase(unittest.TestCase): - """TestCase subclass for testing IOLoop-based asynchronous code. + """`~unittest.TestCase` subclass for testing `.IOLoop`-based + asynchronous code. - The unittest framework is synchronous, so the test must be complete - by the time the test method returns. This method provides the stop() - and wait() methods for this purpose. The test method itself must call - self.wait(), and asynchronous callbacks should call self.stop() to signal - completion. + The unittest framework is synchronous, so the test must be + complete by the time the test method returns. This class provides + the `stop()` and `wait()` methods for this purpose. The test + method itself must call ``self.wait()``, and asynchronous + callbacks should call ``self.stop()`` to signal completion. + Alternately, the `gen_test` decorator can be used to use yield points + from the `tornado.gen` module. - By default, a new IOLoop is constructed for each test and is available - as self.io_loop. This IOLoop should be used in the construction of + By default, a new `.IOLoop` is constructed for each test and is available + as ``self.io_loop``. This `.IOLoop` should be used in the construction of HTTP clients/servers, etc. If the code being tested requires a - global IOLoop, subclasses should override get_new_ioloop to return it. + global `.IOLoop`, subclasses should override `get_new_ioloop` to return it. - The IOLoop's start and stop methods should not be called directly. - Instead, use self.stop self.wait. Arguments passed to self.stop are - returned from self.wait. It is possible to have multiple - wait/stop cycles in the same test. + The `.IOLoop`'s ``start`` and ``stop`` methods should not be + called directly. Instead, use `self.stop ` and `self.wait + `. Arguments passed to ``self.stop`` are returned from + ``self.wait``. It is possible to have multiple ``wait``/``stop`` + cycles in the same test. Example:: - # This test uses an asynchronous style similar to most async - # application code. + # This test uses argument passing between self.stop and self.wait. class MyTestCase(AsyncTestCase): + def test_http_fetch(self): + client = AsyncHTTPClient(self.io_loop) + client.fetch("http://www.tornadoweb.org/", self.stop) + response = self.wait() + # Test contents of response + self.assertIn("FriendFeed", response.body) + + # This test uses an explicit callback-based style. + class MyTestCase2(AsyncTestCase): def test_http_fetch(self): client = AsyncHTTPClient(self.io_loop) client.fetch("http://www.tornadoweb.org/", self.handle_fetch) @@ -128,19 +142,6 @@ class AsyncTestCase(unittest.TestCase): # self.wait() in test_http_fetch() via stack_context. self.assertIn("FriendFeed", response.body) self.stop() - - # This test uses the argument passing between self.stop and self.wait - # for a simpler, more synchronous style. - # This style is recommended over the preceding example because it - # keeps the assertions in the test method itself, and is therefore - # less sensitive to the subtleties of stack_context. - class MyTestCase2(AsyncTestCase): - def test_http_fetch(self): - client = AsyncHTTPClient(self.io_loop) - client.fetch("http://www.tornadoweb.org/", self.stop) - response = self.wait() - # Test contents of response - self.assertIn("FriendFeed", response.body) """ def __init__(self, *args, **kwargs): super(AsyncTestCase, self).__init__(*args, **kwargs) @@ -165,16 +166,21 @@ class AsyncTestCase(unittest.TestCase): # set FD_CLOEXEC on its file descriptors) self.io_loop.close(all_fds=True) super(AsyncTestCase, self).tearDown() + # In case an exception escaped or the StackContext caught an exception + # when there wasn't a wait() to re-raise it, do so here. + # This is our last chance to raise an exception in a way that the + # unittest machinery understands. + self.__rethrow() def get_new_ioloop(self): - """Creates a new IOLoop for this test. May be overridden in - subclasses for tests that require a specific IOLoop (usually - the singleton). + """Creates a new `.IOLoop` for this test. May be overridden in + subclasses for tests that require a specific `.IOLoop` (usually + the singleton `.IOLoop.instance()`). """ return IOLoop() def _handle_exception(self, typ, value, tb): - self.__failure = sys.exc_info() + self.__failure = (typ, value, tb) self.stop() return True @@ -187,16 +193,18 @@ class AsyncTestCase(unittest.TestCase): def run(self, result=None): with ExceptionStackContext(self._handle_exception): super(AsyncTestCase, self).run(result) - # In case an exception escaped super.run or the StackContext caught - # an exception when there wasn't a wait() to re-raise it, do so here. + # As a last resort, if an exception escaped super.run() and wasn't + # re-raised in tearDown, raise it here. This will cause the + # unittest run to fail messily, but that's better than silently + # ignoring an error. self.__rethrow() def stop(self, _arg=None, **kwargs): - """Stops the ioloop, causing one pending (or future) call to wait() + """Stops the `.IOLoop`, causing one pending (or future) call to `wait()` to return. - Keyword arguments or a single positional argument passed to stop() are - saved and will be returned by wait(). + Keyword arguments or a single positional argument passed to `stop()` are + saved and will be returned by `wait()`. """ assert _arg is None or not kwargs self.__stop_args = kwargs or _arg @@ -205,14 +213,19 @@ class AsyncTestCase(unittest.TestCase): self.__running = False self.__stopped = True - def wait(self, condition=None, timeout=5): - """Runs the IOLoop until stop is called or timeout has passed. + def wait(self, condition=None, timeout=None): + """Runs the `.IOLoop` until stop is called or timeout has passed. - In the event of a timeout, an exception will be thrown. + In the event of a timeout, an exception will be thrown. The default + timeout is 5 seconds; it may be overridden with a ``timeout`` keyword + argument or globally with the ASYNC_TEST_TIMEOUT environment variable. - If condition is not None, the IOLoop will be restarted after stop() - until condition() returns true. + If ``condition`` is not None, the `.IOLoop` will be restarted + after `stop()` until ``condition()`` returns true. """ + if timeout is None: + timeout = get_async_test_timeout() + if not self.__stopped: if timeout: def timeout_func(): @@ -244,9 +257,9 @@ class AsyncTestCase(unittest.TestCase): class AsyncHTTPTestCase(AsyncTestCase): """A test case that starts up an HTTP server. - Subclasses must override get_app(), which returns the - tornado.web.Application (or other HTTPServer callback) to be tested. - Tests will typically use the provided self.http_client to fetch + Subclasses must override `get_app()`, which returns the + `tornado.web.Application` (or other `.HTTPServer` callback) to be tested. + Tests will typically use the provided ``self.http_client`` to fetch URLs from this server. Example:: @@ -283,17 +296,17 @@ class AsyncHTTPTestCase(AsyncTestCase): def get_app(self): """Should be overridden by subclasses to return a - tornado.web.Application or other HTTPServer callback. + `tornado.web.Application` or other `.HTTPServer` callback. """ raise NotImplementedError() def fetch(self, path, **kwargs): """Convenience method to synchronously fetch a url. - The given path will be appended to the local server's host and port. - Any additional kwargs will be passed directly to - AsyncHTTPClient.fetch (and so could be used to pass method="POST", - body="...", etc). + The given path will be appended to the local server's host and + port. Any additional kwargs will be passed directly to + `.AsyncHTTPClient.fetch` (and so could be used to pass + ``method="POST"``, ``body="..."``, etc). """ self.http_client.fetch(self.get_url(path), self.stop, **kwargs) return self.wait() @@ -357,34 +370,58 @@ class AsyncHTTPSTestCase(AsyncHTTPTestCase): return 'https' -def gen_test(f): - """Testing equivalent of ``@gen.engine``, to be applied to test methods. +def gen_test(func=None, timeout=None): + """Testing equivalent of ``@gen.coroutine``, to be applied to test methods. - ``@gen.engine`` cannot be used on tests because the `IOLoop` is not + ``@gen.coroutine`` cannot be used on tests because the `.IOLoop` is not already running. ``@gen_test`` should be applied to test methods on subclasses of `AsyncTestCase`. - Note that unlike most uses of ``@gen.engine``, ``@gen_test`` can - detect automatically when the function finishes cleanly so there - is no need to run a callback to signal completion. - Example:: + class MyTest(AsyncHTTPTestCase): @gen_test def test_something(self): response = yield gen.Task(self.fetch('/')) + By default, ``@gen_test`` times out after 5 seconds. The timeout may be + overridden globally with the ASYNC_TEST_TIMEOUT environment variable, + or for each test with the ``timeout`` keyword argument:: + + class MyTest(AsyncHTTPTestCase): + @gen_test(timeout=10) + def test_something_slow(self): + response = yield gen.Task(self.fetch('/')) + + If both the environment variable and the parameter are set, ``gen_test`` + uses the maximum of the two. """ - @functools.wraps(f) - def wrapper(self, *args, **kwargs): - result = f(self, *args, **kwargs) - if result is None: - return - assert isinstance(result, types.GeneratorType) - runner = gen.Runner(result, self.stop) - runner.run() - self.wait() - return wrapper + if timeout is None: + timeout = get_async_test_timeout() + + def wrap(f): + f = gen.coroutine(f) + + @functools.wraps(f) + def wrapper(self): + return self.io_loop.run_sync( + functools.partial(f, self), timeout=timeout) + return wrapper + + if func is not None: + # Used like: + # @gen_test + # def f(self): + # pass + return wrap(func) + else: + # Used like @gen_test(timeout=10) + return wrap + + +# Without this attribute, nosetests will try to run gen_test as a test +# anywhere it is imported. +gen_test.__test__ = False class LogTrapTestCase(unittest.TestCase): @@ -395,13 +432,13 @@ class LogTrapTestCase(unittest.TestCase): the test succeeds, so this class can be useful to minimize the noise. Simply use it as a base class for your test case. It is safe to combine with AsyncTestCase via multiple inheritance - ("class MyTestCase(AsyncHTTPTestCase, LogTrapTestCase):") + (``class MyTestCase(AsyncHTTPTestCase, LogTrapTestCase):``) - This class assumes that only one log handler is configured and that - it is a StreamHandler. This is true for both logging.basicConfig - and the "pretty logging" configured by tornado.options. It is not - compatible with other log buffering mechanisms, such as those provided - by some test runners. + This class assumes that only one log handler is configured and + that it is a `~logging.StreamHandler`. This is true for both + `logging.basicConfig` and the "pretty logging" configured by + `tornado.options`. It is not compatible with other log buffering + mechanisms, such as those provided by some test runners. """ def run(self, result=None): logger = logging.getLogger() @@ -409,7 +446,7 @@ class LogTrapTestCase(unittest.TestCase): logging.basicConfig() handler = logger.handlers[0] if (len(logger.handlers) > 1 or - not isinstance(handler, logging.StreamHandler)): + not isinstance(handler, logging.StreamHandler)): # Logging has been configured in a way we don't recognize, # so just leave it alone. super(LogTrapTestCase, self).run(result) @@ -486,10 +523,11 @@ def main(**kwargs): be specified. Projects with many tests may wish to define a test script like - tornado/test/runtests.py. This script should define a method all() - which returns a test suite and then call tornado.testing.main(). - Note that even when a test script is used, the all() test suite may - be overridden by naming a single test on the command line:: + ``tornado/test/runtests.py``. This script should define a method + ``all()`` which returns a test suite and then call + `tornado.testing.main()`. Note that even when a test script is + used, the ``all()`` test suite may be overridden by naming a + single test on the command line:: # Runs all tests python -m tornado.test.runtests diff --git a/libs/tornado/util.py b/libs/tornado/util.py index 69de2c8e..a2fba779 100755 --- a/libs/tornado/util.py +++ b/libs/tornado/util.py @@ -1,4 +1,14 @@ -"""Miscellaneous utility functions.""" +"""Miscellaneous utility functions and classes. + +This module is used internally by Tornado. It is not necessarily expected +that the functions and classes defined here will be useful to other +applications, but they are documented here in case they are. + +The one public-facing part of this module is the `Configurable` class +and its `~Configurable.configure` method, which becomes a part of the +interface of its subclasses, including `.AsyncHTTPClient`, `.IOLoop`, +and `.Resolver`. +""" from __future__ import absolute_import, division, print_function, with_statement @@ -8,7 +18,8 @@ import zlib class ObjectDict(dict): - """Makes a dictionary behave like an object.""" + """Makes a dictionary behave like an object, with attribute-style access. + """ def __getattr__(self, name): try: return self[name] @@ -52,6 +63,7 @@ class GzipDecompressor(object): def import_object(name): """Imports an object by name. + import_object('x') is equivalent to 'import x'. import_object('x.y.z') is equivalent to 'from x.y import z'. >>> import tornado.escape @@ -59,10 +71,23 @@ def import_object(name): True >>> import_object('tornado.escape.utf8') is tornado.escape.utf8 True + >>> import_object('tornado') is tornado + True + >>> import_object('tornado.missing_module') + Traceback (most recent call last): + ... + ImportError: No module named missing_module """ + if name.count('.') == 0: + return __import__(name, None, None) + parts = name.split('.') obj = __import__('.'.join(parts[:-1]), None, None, [parts[-1]], 0) - return getattr(obj, parts[-1]) + try: + return getattr(obj, parts[-1]) + except AttributeError: + raise ImportError("No module named %s" % parts[-1]) + # Fake unicode literal support: Python 3.2 doesn't have the u'' marker for # literal strings, and alternative solutions like "from __future__ import @@ -115,16 +140,17 @@ class Configurable(object): The implementation subclass as well as optional keyword arguments to its initializer can be set globally at runtime with `configure`. - By using the constructor as the factory method, the interface looks like - a normal class, ``isinstance()`` works as usual, etc. This pattern - is most useful when the choice of implementation is likely to be a - global decision (e.g. when epoll is available, always use it instead of - select), or when a previously-monolithic class has been split into - specialized subclasses. + By using the constructor as the factory method, the interface + looks like a normal class, `isinstance` works as usual, etc. This + pattern is most useful when the choice of implementation is likely + to be a global decision (e.g. when `~select.epoll` is available, + always use it instead of `~select.select`), or when a + previously-monolithic class has been split into specialized + subclasses. Configurable subclasses must define the class methods `configurable_base` and `configurable_default`, and use the instance - method `initialize` instead of `__init__`. + method `initialize` instead of ``__init__``. """ __impl_class = None __impl_kwargs = None @@ -163,7 +189,7 @@ class Configurable(object): def initialize(self): """Initialize a `Configurable` subclass instance. - Configurable classes should use `initialize` instead of `__init__`. + Configurable classes should use `initialize` instead of ``__init__``. """ @classmethod @@ -210,8 +236,6 @@ class ArgReplacer(object): and similar wrappers. """ def __init__(self, func, name): - """Create an ArgReplacer for the named argument to the given function. - """ self.name = name try: self.arg_pos = inspect.getargspec(func).args.index(self.name) diff --git a/libs/tornado/web.py b/libs/tornado/web.py index 3cccba56..6abf42c0 100755 --- a/libs/tornado/web.py +++ b/libs/tornado/web.py @@ -14,13 +14,12 @@ # License for the specific language governing permissions and limitations # under the License. -""" -The Tornado web framework looks a bit like web.py (http://webpy.org/) or -Google's webapp (http://code.google.com/appengine/docs/python/tools/webapp/), -but with additional tools and optimizations to take advantage of the -Tornado non-blocking web server and tools. +"""``tornado.web`` provides a simple web framework with asynchronous +features that allow it to scale to large numbers of open connections, +making it ideal for `long polling +`_. -Here is the canonical "Hello, world" example app:: +Here is a simple "Hello, world" example app:: import tornado.ioloop import tornado.web @@ -36,17 +35,19 @@ Here is the canonical "Hello, world" example app:: application.listen(8888) tornado.ioloop.IOLoop.instance().start() -See the Tornado walkthrough on http://tornadoweb.org for more details -and a good getting started guide. +See the :doc:`Tornado overview ` for more details and a good getting +started guide. Thread-safety notes ------------------- -In general, methods on RequestHandler and elsewhere in tornado are not -thread-safe. In particular, methods such as write(), finish(), and -flush() must only be called from the main thread. If you use multiple -threads it is important to use IOLoop.add_callback to transfer control -back to the main thread before finishing the request. +In general, methods on `RequestHandler` and elsewhere in Tornado are +not thread-safe. In particular, methods such as +`~RequestHandler.write()`, `~RequestHandler.finish()`, and +`~RequestHandler.flush()` must only be called from the main thread. If +you use multiple threads it is important to use `.IOLoop.add_callback` +to transfer control back to the main thread before finishing the +request. """ from __future__ import absolute_import, division, print_function, with_statement @@ -72,6 +73,7 @@ import traceback import types import uuid +from tornado.concurrent import Future from tornado import escape from tornado import httputil from tornado import locale @@ -103,11 +105,11 @@ except ImportError: class RequestHandler(object): - """Subclass this class and define get() or post() to make a handler. + """Subclass this class and define `get()` or `post()` to make a handler. If you want to support more methods than the standard GET/HEAD/POST, you - should override the class variable SUPPORTED_METHODS in your - RequestHandler class. + should override the class variable ``SUPPORTED_METHODS`` in your + `RequestHandler` subclass. """ SUPPORTED_METHODS = ("GET", "HEAD", "POST", "DELETE", "PATCH", "PUT", "OPTIONS") @@ -167,7 +169,7 @@ class RequestHandler(object): @property def settings(self): - """An alias for `self.application.settings`.""" + """An alias for `self.application.settings `.""" return self.application.settings def head(self, *args, **kwargs): @@ -192,7 +194,7 @@ class RequestHandler(object): raise HTTPError(405) def prepare(self): - """Called at the beginning of a request before `get`/`post`/etc. + """Called at the beginning of a request before `get`/`post`/etc. Override this method to perform common initialization regardless of the request method. @@ -228,10 +230,10 @@ class RequestHandler(object): def clear(self): """Resets all headers and content for this response.""" self._headers = httputil.HTTPHeaders({ - "Server": "TornadoServer/%s" % tornado.version, - "Content-Type": "text/html; charset=UTF-8", - "Date": httputil.format_timestamp(time.gmtime()), - }) + "Server": "TornadoServer/%s" % tornado.version, + "Content-Type": "text/html; charset=UTF-8", + "Date": httputil.format_timestamp(time.gmtime()), + }) self.set_default_headers() if not self.request.supports_http_1_1(): if self.request.headers.get("Connection") == "Keep-Alive": @@ -253,10 +255,11 @@ class RequestHandler(object): def set_status(self, status_code, reason=None): """Sets the status code for our response. - :arg int status_code: Response status code. If `reason` is ``None``, - it must be present in `httplib.responses`. + :arg int status_code: Response status code. If ``reason`` is ``None``, + it must be present in `httplib.responses `. :arg string reason: Human-readable reason phrase describing the status - code. If ``None``, it will be filled in from `httplib.responses`. + code. If ``None``, it will be filled in from + `httplib.responses `. """ self._status_code = status_code if reason is not None: @@ -363,8 +366,8 @@ class RequestHandler(object): By default, this method decodes the argument as utf-8 and returns a unicode string, but this may be overridden in subclasses. - This method is used as a filter for both get_argument() and for - values extracted from the url and passed to get()/post()/etc. + This method is used as a filter for both `get_argument()` and for + values extracted from the url and passed to `get()`/`post()`/etc. The name of the argument is provided if known, but may be None (e.g. for unnamed groups in the url regex). @@ -373,6 +376,7 @@ class RequestHandler(object): @property def cookies(self): + """An alias for `self.request.cookies <.httpserver.HTTPRequest.cookies>`.""" return self.request.cookies def get_cookie(self, name, default=None): @@ -496,8 +500,8 @@ class RequestHandler(object): To write the output to the network, use the flush() method below. If the given chunk is a dictionary, we write it as JSON and set - the Content-Type of the response to be application/json. - (if you want to send JSON as a different Content-Type, call + the Content-Type of the response to be ``application/json``. + (if you want to send JSON as a different ``Content-Type``, call set_header *after* calling write()). Note that lists are not converted to JSON because of a potential @@ -604,8 +608,8 @@ class RequestHandler(object): def render_string(self, template_name, **kwargs): """Generate the given template with the given arguments. - We return the generated string. To generate and write a template - as a response, use render() above. + We return the generated byte string (in utf8). To generate and + write a template as a response, use render() above. """ # If no template_path is specified, use the path of the calling file template_path = self.get_template_path() @@ -743,6 +747,9 @@ class RequestHandler(object): self._log() self._finished = True self.on_finish() + # Break up a reference cycle between this handler and the + # _ui_module closures to allow for faster GC on CPython. + self.ui = None def send_error(self, status_code=500, **kwargs): """Sends the given HTTP error code to the browser. @@ -823,9 +830,9 @@ class RequestHandler(object): def locale(self): """The local for the current session. - Determined by either get_user_locale, which you can override to + Determined by either `get_user_locale`, which you can override to set the locale based on, e.g., a user preference stored in a - database, or get_browser_locale, which uses the Accept-Language + database, or `get_browser_locale`, which uses the ``Accept-Language`` header. """ if not hasattr(self, "_locale"): @@ -838,15 +845,15 @@ class RequestHandler(object): def get_user_locale(self): """Override to determine the locale from the authenticated user. - If None is returned, we fall back to get_browser_locale(). + If None is returned, we fall back to `get_browser_locale()`. - This method should return a tornado.locale.Locale object, - most likely obtained via a call like tornado.locale.get("en") + This method should return a `tornado.locale.Locale` object, + most likely obtained via a call like ``tornado.locale.get("en")`` """ return None def get_browser_locale(self, default="en_US"): - """Determines the user's locale from Accept-Language header. + """Determines the user's locale from ``Accept-Language`` header. See http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.4 """ @@ -873,9 +880,9 @@ class RequestHandler(object): def current_user(self): """The authenticated user for this request. - Determined by either get_current_user, which you can override to - set the user based on, e.g., a cookie. If that method is not - overridden, this method always returns None. + This is a cached version of `get_current_user`, which you can + override to set the user based on, e.g., a cookie. If that + method is not overridden, this method always returns None. We lazy-load the current user the first time this method is called and cache the result after that. @@ -891,7 +898,7 @@ class RequestHandler(object): def get_login_url(self): """Override to customize the login URL based on the request. - By default, we use the 'login_url' application setting. + By default, we use the ``login_url`` application setting. """ self.require_setting("login_url", "@tornado.web.authenticated") return self.application.settings["login_url"] @@ -899,7 +906,7 @@ class RequestHandler(object): def get_template_path(self): """Override to customize template path for each handler. - By default, we use the 'template_path' application setting. + By default, we use the ``template_path`` application setting. Return None to load templates relative to the calling file. """ return self.application.settings.get("template_path") @@ -925,21 +932,21 @@ class RequestHandler(object): return self._xsrf_token def check_xsrf_cookie(self): - """Verifies that the '_xsrf' cookie matches the '_xsrf' argument. + """Verifies that the ``_xsrf`` cookie matches the ``_xsrf`` argument. - To prevent cross-site request forgery, we set an '_xsrf' + To prevent cross-site request forgery, we set an ``_xsrf`` cookie and include the same value as a non-cookie - field with all POST requests. If the two do not match, we + field with all ``POST`` requests. If the two do not match, we reject the form submission as a potential forgery. - The _xsrf value may be set as either a form field named _xsrf - or in a custom HTTP header named X-XSRFToken or X-CSRFToken + The ``_xsrf`` value may be set as either a form field named ``_xsrf`` + or in a custom HTTP header named ``X-XSRFToken`` or ``X-CSRFToken`` (the latter is accepted for compatibility with Django). See http://en.wikipedia.org/wiki/Cross-site_request_forgery Prior to release 1.1.1, this check was ignored if the HTTP header - "X-Requested-With: XMLHTTPRequest" was present. This exception + ``X-Requested-With: XMLHTTPRequest`` was present. This exception has been shown to be insecure and has been removed. For more information please see http://www.djangoproject.com/weblog/2011/feb/08/security/ @@ -954,14 +961,17 @@ class RequestHandler(object): raise HTTPError(403, "XSRF cookie does not match POST argument") def xsrf_form_html(self): - """An HTML element to be included with all POST forms. + """An HTML ```` element to be included with all POST forms. - It defines the _xsrf input value, which we check on all POST + It defines the ``_xsrf`` input value, which we check on all POST requests to prevent cross-site request forgery. If you have set - the 'xsrf_cookies' application setting, you must include this + the ``xsrf_cookies`` application setting, you must include this HTML within all of your HTML forms. - See check_xsrf_cookie() above for more information. + In a template, this method should be called with ``{% module + xsrf_form_html() %}`` + + See `check_xsrf_cookie()` above for more information. """ return '' @@ -969,11 +979,11 @@ class RequestHandler(object): def static_url(self, path, include_host=None): """Returns a static URL for the given relative static file path. - This method requires you set the 'static_path' setting in your + This method requires you set the ``static_path`` setting in your application (which specifies the root directory of your static files). - We append ?v= to the returned URL, which makes our + We append ``?v=`` to the returned URL, which makes our static file handler set an infinite expiration header on the returned content. The signature is based on the content of the file. @@ -1142,10 +1152,18 @@ class RequestHandler(object): def asynchronous(method): """Wrap request handler methods with this if they are asynchronous. + This decorator should only be applied to the :ref:`HTTP verb + methods `; its behavior is undefined for any other method. + This decorator does not *make* a method asynchronous; it tells + the framework that the method *is* asynchronous. For this decorator + to be useful the method must (at least sometimes) do something + asynchronous. + If this decorator is given, the response is not finished when the - method returns. It is up to the request handler to call self.finish() - to finish the HTTP request. Without this decorator, the request is - automatically finished when the get() or post() method returns. :: + method returns. It is up to the request handler to call + `self.finish() ` to finish the HTTP + request. Without this decorator, the request is automatically + finished when the ``get()`` or ``post()`` method returns. Example:: class MyRequestHandler(web.RequestHandler): @web.asynchronous @@ -1158,6 +1176,8 @@ def asynchronous(method): self.finish() """ + # Delay the IOLoop import because it's not available on app engine. + from tornado.ioloop import IOLoop @functools.wraps(method) def wrapper(self, *args, **kwargs): if self.application._wsgi: @@ -1165,14 +1185,27 @@ def asynchronous(method): self._auto_finish = False with stack_context.ExceptionStackContext( self._stack_context_handle_exception): - return method(self, *args, **kwargs) + result = method(self, *args, **kwargs) + if isinstance(result, Future): + # If @asynchronous is used with @gen.coroutine, (but + # not @gen.engine), we can automatically finish the + # request when the future resolves. Additionally, + # the Future will swallow any exceptions so we need + # to throw them back out to the stack context to finish + # the request. + def future_complete(f): + f.result() + if not self._finished: + self.finish() + IOLoop.current().add_future(result, future_complete) + return result return wrapper def removeslash(method): """Use this decorator to remove trailing slashes from the request path. - For example, a request to ``'/foo/'`` would redirect to ``'/foo'`` with this + For example, a request to ``/foo/`` would redirect to ``/foo`` with this decorator. Your request handler mapping should use a regular expression like ``r'/foo/*'`` in conjunction with using the decorator. """ @@ -1195,9 +1228,9 @@ def removeslash(method): def addslash(method): """Use this decorator to add a missing trailing slash to the request path. - For example, a request to '/foo' would redirect to '/foo/' with this + For example, a request to ``/foo`` would redirect to ``/foo/`` with this decorator. Your request handler mapping should use a regular expression - like r'/foo/?' in conjunction with using the decorator. + like ``r'/foo/?'`` in conjunction with using the decorator. """ @functools.wraps(method) def wrapper(self, *args, **kwargs): @@ -1226,35 +1259,36 @@ class Application(object): http_server.listen(8080) ioloop.IOLoop.instance().start() - The constructor for this class takes in a list of URLSpec objects + The constructor for this class takes in a list of `URLSpec` objects or (regexp, request_class) tuples. When we receive requests, we iterate over the list in order and instantiate an instance of the first request class whose regexp matches the request path. - Each tuple can contain an optional third element, which should be a - dictionary if it is present. That dictionary is passed as keyword - arguments to the contructor of the handler. This pattern is used - for the StaticFileHandler below (note that a StaticFileHandler - can be installed automatically with the static_path setting described - below):: + Each tuple can contain an optional third element, which should be + a dictionary if it is present. That dictionary is passed as + keyword arguments to the contructor of the handler. This pattern + is used for the `StaticFileHandler` in this example (note that a + `StaticFileHandler` can be installed automatically with the + static_path setting described below):: application = web.Application([ (r"/static/(.*)", web.StaticFileHandler, {"path": "/var/www"}), ]) - We support virtual hosts with the add_handlers method, which takes in + We support virtual hosts with the `add_handlers` method, which takes in a host regular expression as the first argument:: application.add_handlers(r"www\.myhost\.com", [ (r"/article/([0-9]+)", ArticleHandler), ]) - You can serve static files by sending the static_path setting as a - keyword argument. We will serve those files from the /static/ URI - (this is configurable with the static_url_prefix setting), - and we will serve /favicon.ico and /robots.txt from the same directory. - A custom subclass of StaticFileHandler can be specified with the - static_handler_class setting. + You can serve static files by sending the ``static_path`` setting + as a keyword argument. We will serve those files from the + ``/static/`` URI (this is configurable with the + ``static_url_prefix`` setting), and we will serve ``/favicon.ico`` + and ``/robots.txt`` from the same directory. A custom subclass of + `StaticFileHandler` can be specified with the + ``static_handler_class`` setting. """ def __init__(self, handlers=None, default_host="", transforms=None, wsgi=False, **settings): @@ -1301,15 +1335,16 @@ class Application(object): def listen(self, port, address="", **kwargs): """Starts an HTTP server for this application on the given port. - This is a convenience alias for creating an HTTPServer object - and calling its listen method. Keyword arguments not - supported by HTTPServer.listen are passed to the HTTPServer - constructor. For advanced uses (e.g. preforking), do not use - this method; create an HTTPServer and call its bind/start - methods directly. + This is a convenience alias for creating an `.HTTPServer` + object and calling its listen method. Keyword arguments not + supported by `HTTPServer.listen <.TCPServer.listen>` are passed to the + `.HTTPServer` constructor. For advanced uses + (e.g. multi-process mode), do not use this method; create an + `.HTTPServer` and call its + `.TCPServer.bind`/`.TCPServer.start` methods directly. Note that after calling this method you still need to call - IOLoop.instance().start() to start the server. + ``IOLoop.instance().start()`` to start the server. """ # import is here rather than top level because HTTPServer # is not importable on appengine @@ -1337,7 +1372,7 @@ class Application(object): self.handlers.append((re.compile(host_pattern), handlers)) for spec in host_handlers: - if isinstance(spec, type(())): + if isinstance(spec, (tuple, list)): assert len(spec) in (2, 3) pattern = spec[0] handler = spec[1] @@ -1361,7 +1396,6 @@ class Application(object): self.named_handlers[spec.name] = spec def add_transform(self, transform_class): - """Adds the given OutputTransform to our transform list.""" self.transforms.append(transform_class) def _get_host_handlers(self, request): @@ -1456,11 +1490,11 @@ class Application(object): return handler def reverse_url(self, name, *args): - """Returns a URL path for handler named `name` + """Returns a URL path for handler named ``name`` - The handler must be added to the application as a named URLSpec. + The handler must be added to the application as a named `URLSpec`. - Args will be substituted for capturing groups in the URLSpec regex. + Args will be substituted for capturing groups in the `URLSpec` regex. They will be converted to strings if necessary, encoded as utf8, and url-escaped. """ @@ -1474,7 +1508,7 @@ class Application(object): By default writes to the python root logger. To change this behavior either subclass Application and override this method, or pass a function in the application settings dictionary as - 'log_function'. + ``log_function``. """ if "log_function" in self.settings: self.settings["log_function"](handler) @@ -1493,8 +1527,13 @@ class Application(object): class HTTPError(Exception): """An exception that will turn into an HTTP error response. + Raising an `HTTPError` is a convenient alternative to calling + `RequestHandler.send_error` since it automatically ends the + current function. + :arg int status_code: HTTP status code. Must be listed in - `httplib.responses` unless the ``reason`` keyword argument is given. + `httplib.responses ` unless the ``reason`` + keyword argument is given. :arg string log_message: Message to be written to the log for this error (will not be shown to the user unless the `Application` is in debug mode). May contain ``%s``-style placeholders, which will be filled @@ -1521,7 +1560,7 @@ class HTTPError(Exception): class ErrorHandler(RequestHandler): - """Generates an error response with status_code for all requests.""" + """Generates an error response with ``status_code`` for all requests.""" def initialize(self, status_code): self.set_status(status_code) @@ -1538,7 +1577,7 @@ class ErrorHandler(RequestHandler): class RedirectHandler(RequestHandler): """Redirects the client to the given URL for all GET requests. - You should provide the keyword argument "url" to the handler, e.g.:: + You should provide the keyword argument ``url`` to the handler, e.g.:: application = web.Application([ (r"/oldpath", web.RedirectHandler, {"url": "/newpath"}), @@ -1555,20 +1594,20 @@ class RedirectHandler(RequestHandler): class StaticFileHandler(RequestHandler): """A simple handler that can serve static content from a directory. - To map a path to this handler for a static data directory /var/www, + To map a path to this handler for a static data directory ``/var/www``, you would add a line to your application like:: application = web.Application([ (r"/static/(.*)", web.StaticFileHandler, {"path": "/var/www"}), ]) - The local root directory of the content should be passed as the "path" + The local root directory of the content should be passed as the ``path`` argument to the handler. - To support aggressive browser caching, if the argument "v" is given + To support aggressive browser caching, if the argument ``v`` is given with the path, we set an infinite HTTP expiration header. So, if you want browsers to cache a file indefinitely, send them to, e.g., - /static/images/myimage.png?v=xxx. Override ``get_cache_time`` method for + ``/static/images/myimage.png?v=xxx``. Override `get_cache_time` method for more fine-grained cache control. """ CACHE_MAX_AGE = 86400 * 365 * 10 # 10 years @@ -1609,7 +1648,7 @@ class StaticFileHandler(RequestHandler): raise HTTPError(403, "%s is not a file", path) stat_result = os.stat(abspath) - modified = datetime.datetime.fromtimestamp(stat_result[stat.ST_MTIME]) + modified = datetime.datetime.utcfromtimestamp(stat_result[stat.ST_MTIME]) self.set_header("Last-Modified", modified) @@ -1631,7 +1670,7 @@ class StaticFileHandler(RequestHandler): ims_value = self.request.headers.get("If-Modified-Since") if ims_value is not None: date_tuple = email.utils.parsedate(ims_value) - if_since = datetime.datetime.fromtimestamp(time.mktime(date_tuple)) + if_since = datetime.datetime(*date_tuple[:6]) if if_since >= modified: self.set_status(304) return @@ -1651,11 +1690,13 @@ class StaticFileHandler(RequestHandler): def get_cache_time(self, path, modified, mime_type): """Override to customize cache control behavior. - Return a positive number of seconds to trigger aggressive caching or 0 - to mark resource as cacheable, only. + Return a positive number of seconds to make the result + cacheable for that amount of time or 0 to mark resource as + cacheable for an unspecified amount of time (subject to + browser heuristics). By default returns cache expiry of 10 years for resources requested - with "v" argument. + with ``v`` argument. """ return self.CACHE_MAX_AGE if "v" in self.request.arguments else 0 @@ -1718,12 +1759,13 @@ class StaticFileHandler(RequestHandler): class FallbackHandler(RequestHandler): - """A RequestHandler that wraps another HTTP server callback. + """A `RequestHandler` that wraps another HTTP server callback. - The fallback is a callable object that accepts an HTTPRequest, - such as an Application or tornado.wsgi.WSGIContainer. This is most - useful to use both tornado RequestHandlers and WSGI in the same server. - Typical usage:: + The fallback is a callable object that accepts an + `~.httpserver.HTTPRequest`, such as an `Application` or + `tornado.wsgi.WSGIContainer`. This is most useful to use both + Tornado ``RequestHandlers`` and WSGI in the same server. Typical + usage:: wsgi_app = tornado.wsgi.WSGIContainer( django.core.handlers.wsgi.WSGIHandler()) @@ -1837,7 +1879,11 @@ class ChunkedTransferEncoding(OutputTransform): def authenticated(method): - """Decorate methods with this to require that the user be logged in.""" + """Decorate methods with this to require that the user be logged in. + + If the user is not logged in, they will be redirected to the configured + `login url `. + """ @functools.wraps(method) def wrapper(self, *args, **kwargs): if not self.current_user: @@ -1858,7 +1904,7 @@ def authenticated(method): class UIModule(object): - """A UI re-usable, modular unit on a page. + """A re-usable, modular UI unit on a page. UI modules often execute additional queries, and they can include additional CSS and JavaScript that will be included in the output @@ -1985,21 +2031,19 @@ class TemplateModule(UIModule): class URLSpec(object): """Specifies mappings between URLs and handlers.""" def __init__(self, pattern, handler_class, kwargs=None, name=None): - """Creates a URLSpec. + """Parameters: - Parameters: + * ``pattern``: Regular expression to be matched. Any groups + in the regex will be passed in to the handler's get/post/etc + methods as arguments. - pattern: Regular expression to be matched. Any groups in the regex - will be passed in to the handler's get/post/etc methods as - arguments. + * ``handler_class``: `RequestHandler` subclass to be invoked. - handler_class: RequestHandler subclass to be invoked. + * ``kwargs`` (optional): A dictionary of additional arguments + to be passed to the handler's constructor. - kwargs (optional): A dictionary of additional arguments to be passed - to the handler's constructor. - - name (optional): A name for this handler. Used by - Application.reverse_url. + * ``name`` (optional): A name for this handler. Used by + `Application.reverse_url`. """ if not pattern.endswith('$'): pattern += '$' diff --git a/libs/tornado/websocket.py b/libs/tornado/websocket.py index 235a1102..7bc65138 100755 --- a/libs/tornado/websocket.py +++ b/libs/tornado/websocket.py @@ -1,4 +1,4 @@ -"""Server-side implementation of the WebSocket protocol. +"""Implementation of the WebSocket protocol. `WebSockets `_ allow for bidirectional communication between the browser and server. @@ -25,46 +25,44 @@ import base64 import collections import functools import hashlib -import logging import os -import re -import socket import struct import time import tornado.escape import tornado.web -from tornado.concurrent import Future, return_future -from tornado.escape import utf8, to_unicode, native_str -from tornado.httputil import HTTPHeaders +from tornado.concurrent import Future +from tornado.escape import utf8, native_str +from tornado import httpclient from tornado.ioloop import IOLoop -from tornado.iostream import IOStream, SSLIOStream from tornado.log import gen_log, app_log from tornado.netutil import Resolver from tornado import simple_httpclient -from tornado.util import bytes_type +from tornado.util import bytes_type, unicode_type try: xrange # py2 except NameError: xrange = range # py3 -try: - import urlparse # py2 -except ImportError: - import urllib.parse as urlparse # py3 + +class WebSocketError(Exception): + pass + class WebSocketHandler(tornado.web.RequestHandler): """Subclass this class to create a basic WebSocket handler. - Override on_message to handle incoming messages. You can also override - open and on_close to handle opened and closed connections. + Override `on_message` to handle incoming messages, and use + `write_message` to send messages to the client. You can also + override `open` and `on_close` to handle opened and closed + connections. See http://dev.w3.org/html5/websockets/ for details on the JavaScript interface. The protocol is specified at http://tools.ietf.org/html/rfc6455. - Here is an example Web Socket handler that echos back all received messages + Here is an example WebSocket handler that echos back all received messages back to the client:: class EchoWebSocket(websocket.WebSocketHandler): @@ -77,14 +75,15 @@ class WebSocketHandler(tornado.web.RequestHandler): def on_close(self): print "WebSocket closed" - Web Sockets are not standard HTTP connections. The "handshake" is HTTP, - but after the handshake, the protocol is message-based. Consequently, - most of the Tornado HTTP facilities are not available in handlers of this - type. The only communication methods available to you are write_message() - and close(). Likewise, your request handler class should - implement open() method rather than get() or post(). + WebSockets are not standard HTTP connections. The "handshake" is + HTTP, but after the handshake, the protocol is + message-based. Consequently, most of the Tornado HTTP facilities + are not available in handlers of this type. The only communication + methods available to you are `write_message()`, `ping()`, and + `close()`. Likewise, your request handler class should implement + `open()` method rather than ``get()`` or ``post()``. - If you map the handler above to "/websocket" in your application, you can + If you map the handler above to ``/websocket`` in your application, you can invoke it in JavaScript with:: var ws = new WebSocket("ws://localhost:8888/websocket"); @@ -211,6 +210,7 @@ class WebSocketHandler(tornado.web.RequestHandler): Once the close handshake is successful the socket will be closed. """ self.ws_connection.close() + self.ws_connection = None def allow_draft76(self): """Override to enable support for the older "draft76" protocol. @@ -240,11 +240,10 @@ class WebSocketHandler(tornado.web.RequestHandler): return "wss" if self.request.protocol == "https" else "ws" def async_callback(self, callback, *args, **kwargs): - """Wrap callbacks with this if they are used on asynchronous requests. + """Obsolete - catches exceptions from the wrapped function. - Catches exceptions properly and closes this WebSocket if an exception - is uncaught. (Note that this is usually unnecessary thanks to - `tornado.stack_context`) + This function is normally unncecessary thanks to + `tornado.stack_context`. """ return self.ws_connection.async_callback(callback, *args, **kwargs) @@ -398,8 +397,9 @@ class WebSocketProtocol76(WebSocketProtocol): """Processes the key headers and calculates their key value. Raises ValueError when feed invalid key.""" + # pyflakes complains about variable reuse if both of these lines use 'c' number = int(''.join(c for c in key if c.isdigit())) - spaces = len([c for c in key if c.isspace()]) + spaces = len([c2 for c2 in key if c2.isspace()]) try: key_number = number // spaces except (ValueError, ZeroDivisionError): @@ -444,7 +444,7 @@ class WebSocketProtocol76(WebSocketProtocol): if binary: raise ValueError( "Binary messages not supported by this version of websockets") - if isinstance(message, unicode): + if isinstance(message, unicode_type): message = message.encode("utf-8") assert isinstance(message, bytes_type) self.stream.write(b"\x00" + message + b"\xff") @@ -520,7 +520,6 @@ class WebSocketProtocol13(WebSocketProtocol): return WebSocketProtocol13.compute_accept_value( self.request.headers.get("Sec-Websocket-Key")) - def _accept_connection(self): subprotocol_header = '' subprotocols = self.request.headers.get("Sec-WebSocket-Protocol", '') @@ -727,7 +726,8 @@ class WebSocketProtocol13(WebSocketProtocol): self.stream.io_loop.time() + 5, self._abort) -class _WebSocketClientConnection(simple_httpclient._HTTPConnection): +class WebSocketClientConnection(simple_httpclient._HTTPConnection): + """WebSocket client connection.""" def __init__(self, io_loop, request): self.connect_future = Future() self.read_future = None @@ -738,19 +738,26 @@ class _WebSocketClientConnection(simple_httpclient._HTTPConnection): scheme = {'ws': 'http', 'wss': 'https'}[scheme] request.url = scheme + sep + rest request.headers.update({ - 'Upgrade': 'websocket', - 'Connection': 'Upgrade', - 'Sec-WebSocket-Key': self.key, - 'Sec-WebSocket-Version': '13', - }) + 'Upgrade': 'websocket', + 'Connection': 'Upgrade', + 'Sec-WebSocket-Key': self.key, + 'Sec-WebSocket-Version': '13', + }) - super(_WebSocketClientConnection, self).__init__( - io_loop, None, request, lambda: None, lambda response: None, + super(WebSocketClientConnection, self).__init__( + io_loop, None, request, lambda: None, self._on_http_response, 104857600, Resolver(io_loop=io_loop)) def _on_close(self): self.on_message(None) + def _on_http_response(self, response): + if not self.connect_future.done(): + if response.error: + self.connect_future.set_exception(response.error) + else: + self.connect_future.set_exception(WebSocketError( + "Non-websocket response")) def _handle_1xx(self, code): assert code == 101 @@ -769,9 +776,17 @@ class _WebSocketClientConnection(simple_httpclient._HTTPConnection): self.connect_future.set_result(self) def write_message(self, message, binary=False): + """Sends a message to the WebSocket server.""" self.protocol.write_message(message, binary) def read_message(self, callback=None): + """Reads a message from the WebSocket server. + + Returns a future whose result is the message, or None + if the connection is closed. If a callback argument + is given it will be called with the future when it is + ready. + """ assert self.read_future is None future = Future() if self.read_queue: @@ -793,13 +808,18 @@ class _WebSocketClientConnection(simple_httpclient._HTTPConnection): pass -def WebSocketConnect(url, io_loop=None, callback=None): +def websocket_connect(url, io_loop=None, callback=None, connect_timeout=None): + """Client-side websocket support. + + Takes a url and returns a Future whose result is a + `WebSocketClientConnection`. + """ if io_loop is None: - io_loop = IOLoop.instance() - request = simple_httpclient.HTTPRequest(url) - request = simple_httpclient._RequestProxy( - request, simple_httpclient.HTTPRequest._DEFAULTS) - conn = _WebSocketClientConnection(io_loop, request) + io_loop = IOLoop.current() + request = httpclient.HTTPRequest(url, connect_timeout=connect_timeout) + request = httpclient._RequestProxy( + request, httpclient.HTTPRequest._DEFAULTS) + conn = WebSocketClientConnection(io_loop, request) if callback is not None: io_loop.add_future(conn.connect_future, callback) return conn.connect_future diff --git a/libs/tornado/wsgi.py b/libs/tornado/wsgi.py index 3d06860f..62cff590 100755 --- a/libs/tornado/wsgi.py +++ b/libs/tornado/wsgi.py @@ -82,11 +82,11 @@ else: class WSGIApplication(web.Application): """A WSGI equivalent of `tornado.web.Application`. - WSGIApplication is very similar to web.Application, except no - asynchronous methods are supported (since WSGI does not support - non-blocking requests properly). If you call self.flush() or other - asynchronous methods in your request handlers running in a - WSGIApplication, we throw an exception. + `WSGIApplication` is very similar to `tornado.web.Application`, + except no asynchronous methods are supported (since WSGI does not + support non-blocking requests properly). If you call + ``self.flush()`` or other asynchronous methods in your request + handlers running in a `WSGIApplication`, we throw an exception. Example usage:: @@ -105,13 +105,15 @@ class WSGIApplication(web.Application): server = wsgiref.simple_server.make_server('', 8888, application) server.serve_forever() - See the 'appengine' demo for an example of using this module to run - a Tornado app on Google AppEngine. + See the `appengine demo + `_ + for an example of using this module to run a Tornado app on Google + App Engine. - Since no asynchronous methods are available for WSGI applications, the - httpclient and auth modules are both not available for WSGI applications. - We support the same interface, but handlers running in a WSGIApplication - do not support flush() or asynchronous methods. + WSGI applications use the same `.RequestHandler` class, but not + ``@asynchronous`` methods or ``flush()``. This means that it is + not possible to use `.AsyncHTTPClient`, or the `tornado.auth` or + `tornado.websocket` modules. """ def __init__(self, handlers=None, default_host="", **settings): web.Application.__init__(self, handlers, default_host, transforms=[], @@ -134,7 +136,7 @@ class WSGIApplication(web.Application): class HTTPRequest(object): """Mimics `tornado.httpserver.HTTPRequest` for WSGI applications.""" def __init__(self, environ): - """Parses the given WSGI environ to construct the request.""" + """Parses the given WSGI environment to construct the request.""" self.method = environ["REQUEST_METHOD"] self.path = urllib_parse.quote(from_wsgi_str(environ.get("SCRIPT_NAME", ""))) self.path += urllib_parse.quote(from_wsgi_str(environ.get("PATH_INFO", ""))) @@ -206,7 +208,7 @@ class HTTPRequest(object): class WSGIContainer(object): r"""Makes a WSGI-compatible function runnable on Tornado's HTTP server. - Wrap a WSGI function in a WSGIContainer and pass it to HTTPServer to + Wrap a WSGI function in a `WSGIContainer` and pass it to `.HTTPServer` to run it. For example:: def simple_app(environ, start_response):