Update tornado

libs/tornado/__init__.py

@@ -16,6 +16,8 @@
 
 """The Tornado web server and tools."""
 
+from __future__ import absolute_import, division, with_statement
+
 # version is a human-readable version number.
 
 # version_info is a four-tuple for programmatic comparison. The first
@@ -23,5 +25,5 @@
 # is zero for an official release, positive for a development branch,
 # or negative for a release candidate (after the base version number
 # has been incremented)
-version = "2.2.1"
+version = "2.3.post1"
-version_info = (2, 2, 1, 0)
+version_info = (2, 3, 0, 1)

libs/tornado/auth.py

@@ -44,6 +44,8 @@ Example usage for Google OpenID::
             # Save the user with, e.g., set_secure_cookie()
 """
 
+from __future__ import absolute_import, division, with_statement
+
 import base64
 import binascii
 import hashlib
@@ -59,13 +61,14 @@ from tornado import escape
 from tornado.httputil import url_concat
 from tornado.util import bytes_type, b
 
+
 class OpenIdMixin(object):
     """Abstract implementation of OpenID and Attribute Exchange.
 
     See GoogleMixin below for example implementations.
     """
     def authenticate_redirect(self, callback_uri=None,
-                              ax_attrs=["name","email","language","username"]):
+                              ax_attrs=["name", "email", "language", "username"]):
         """Returns the authentication URL for this service.
 
         After authentication, the service will redirect back to the given
@@ -91,7 +94,8 @@ class OpenIdMixin(object):
         args = dict((k, v[-1]) for k, v in self.request.arguments.iteritems())
         args["openid.mode"] = u"check_authentication"
         url = self._OPENID_ENDPOINT
-        if http_client is None: http_client = httpclient.AsyncHTTPClient()
+        if http_client is None:
+            http_client = httpclient.AsyncHTTPClient()
         http_client.fetch(url, self.async_callback(
             self._on_authentication_verified, callback),
             method="POST", body=urllib.urlencode(args))
@@ -158,8 +162,10 @@ class OpenIdMixin(object):
                self.get_argument(name) == u"http://openid.net/srv/ax/1.0":
                 ax_ns = name[10:]
                 break
+
         def get_ax_arg(uri):
-            if not ax_ns: return u""
+            if not ax_ns:
+                return u""
             prefix = "openid." + ax_ns + ".type."
             ax_name = None
             for name in self.request.arguments.iterkeys():
@@ -167,7 +173,8 @@ class OpenIdMixin(object):
                     part = name[len(prefix):]
                     ax_name = "openid." + ax_ns + ".value." + part
                     break
-            if not ax_name: return u""
+            if not ax_name:
+                return u""
             return self.get_argument(ax_name, u"")
 
         email = get_ax_arg("http://axschema.org/contact/email")
@@ -190,9 +197,12 @@ class OpenIdMixin(object):
             user["name"] = u" ".join(name_parts)
         elif email:
             user["name"] = email.split("@")[0]
-        if email: user["email"] = email
+        if email:
-        if locale: user["locale"] = locale
+            user["email"] = email
-        if username: user["username"] = username
+        if locale:
+            user["locale"] = locale
+        if username:
+            user["username"] = username
         callback(user)
 
 
@@ -235,7 +245,6 @@ class OAuthMixin(object):
                     self._on_request_token, self._OAUTH_AUTHORIZE_URL,
                     callback_uri))
 
-
     def get_authenticated_user(self, callback, http_client=None):
         """Gets the OAuth authorized user and access token on callback.
 
@@ -269,7 +278,7 @@ class OAuthMixin(object):
         http_client.fetch(self._oauth_access_token_url(token),
                           self.async_callback(self._on_access_token, callback))
 
-    def _oauth_request_token_url(self, callback_uri= None, extra_params=None):
+    def _oauth_request_token_url(self, callback_uri=None, extra_params=None):
         consumer_token = self._oauth_consumer_token()
         url = self._OAUTH_REQUEST_TOKEN_URL
         args = dict(
@@ -283,7 +292,8 @@ class OAuthMixin(object):
             if callback_uri:
                 args["oauth_callback"] = urlparse.urljoin(
                     self.request.full_url(), callback_uri)
-            if extra_params: args.update(extra_params)
+            if extra_params:
+                args.update(extra_params)
             signature = _oauth10a_signature(consumer_token, "GET", url, args)
         else:
             signature = _oauth_signature(consumer_token, "GET", url, args)
@@ -316,7 +326,7 @@ class OAuthMixin(object):
             oauth_version=getattr(self, "_OAUTH_VERSION", "1.0a"),
         )
         if "verifier" in request_token:
-          args["oauth_verifier"]=request_token["verifier"]
+            args["oauth_verifier"] = request_token["verifier"]
 
         if getattr(self, "_OAUTH_VERSION", "1.0a") == "1.0a":
             signature = _oauth10a_signature(consumer_token, "GET", url, args,
@@ -376,11 +386,12 @@ class OAuthMixin(object):
         base_args["oauth_signature"] = signature
         return base_args
 
+
 class OAuth2Mixin(object):
     """Abstract implementation of OAuth v 2."""
 
     def authorize_redirect(self, redirect_uri=None, client_id=None,
-                           client_secret=None, extra_params=None ):
+                           client_secret=None, extra_params=None):
         """Redirects the user to obtain OAuth authorization for this service.
 
         Some providers require that you register a Callback
@@ -393,11 +404,12 @@ class OAuth2Mixin(object):
           "redirect_uri": redirect_uri,
           "client_id": client_id
         }
-        if extra_params: args.update(extra_params)
+        if extra_params:
+            args.update(extra_params)
         self.redirect(
                 url_concat(self._OAUTH_AUTHORIZE_URL, args))
 
-    def _oauth_request_token_url(self, redirect_uri= None, client_id = None,
+    def _oauth_request_token_url(self, redirect_uri=None, client_id=None,
                                  client_secret=None, code=None,
                                  extra_params=None):
         url = self._OAUTH_ACCESS_TOKEN_URL
@@ -407,9 +419,11 @@ class OAuth2Mixin(object):
             client_id=client_id,
             client_secret=client_secret,
             )
-        if extra_params: args.update(extra_params)
+        if extra_params:
+            args.update(extra_params)
         return url_concat(url, args)
 
+
 class TwitterMixin(OAuthMixin):
     """Twitter OAuth authentication.
 
@@ -450,15 +464,14 @@ class TwitterMixin(OAuthMixin):
     _OAUTH_AUTHENTICATE_URL = "http://api.twitter.com/oauth/authenticate"
     _OAUTH_NO_CALLBACKS = False
 
-
+    def authenticate_redirect(self, callback_uri=None):
-    def authenticate_redirect(self, callback_uri = None):
         """Just like authorize_redirect(), but auto-redirects if authorized.
 
         This is generally the right interface to use if you are using
         Twitter for single-sign on.
         """
         http = httpclient.AsyncHTTPClient()
-        http.fetch(self._oauth_request_token_url(callback_uri = callback_uri), self.async_callback(
+        http.fetch(self._oauth_request_token_url(callback_uri=callback_uri), self.async_callback(
             self._on_request_token, self._OAUTH_AUTHENTICATE_URL, None))
 
     def twitter_request(self, path, callback, access_token=None,
@@ -514,7 +527,8 @@ class TwitterMixin(OAuthMixin):
             oauth = self._oauth_request_parameters(
                 url, access_token, all_args, method=method)
             args.update(oauth)
-        if args: url += "?" + urllib.urlencode(args)
+        if args:
+            url += "?" + urllib.urlencode(args)
         callback = self.async_callback(self._on_twitter_request, callback)
         http = httpclient.AsyncHTTPClient()
         if post_args is not None:
@@ -590,7 +604,6 @@ class FriendFeedMixin(OAuthMixin):
     _OAUTH_NO_CALLBACKS = True
     _OAUTH_VERSION = "1.0"
 
-
     def friendfeed_request(self, path, callback, access_token=None,
                            post_args=None, **args):
         """Fetches the given relative API path, e.g., "/bret/friends"
@@ -636,7 +649,8 @@ class FriendFeedMixin(OAuthMixin):
             oauth = self._oauth_request_parameters(
                 url, access_token, all_args, method=method)
             args.update(oauth)
-        if args: url += "?" + urllib.urlencode(args)
+        if args:
+            url += "?" + urllib.urlencode(args)
         callback = self.async_callback(self._on_friendfeed_request, callback)
         http = httpclient.AsyncHTTPClient()
         if post_args is not None:
@@ -701,7 +715,7 @@ class GoogleMixin(OpenIdMixin, OAuthMixin):
     _OAUTH_ACCESS_TOKEN_URL = "https://www.google.com/accounts/OAuthGetAccessToken"
 
     def authorize_redirect(self, oauth_scope, callback_uri=None,
-                           ax_attrs=["name","email","language","username"]):
+                           ax_attrs=["name", "email", "language", "username"]):
         """Authenticates and authorizes for the given Google resource.
 
         Some of the available resources are:
@@ -746,6 +760,7 @@ class GoogleMixin(OpenIdMixin, OAuthMixin):
     def _oauth_get_user(self, access_token, callback):
         OpenIdMixin.get_authenticated_user(self, callback)
 
+
 class FacebookMixin(object):
     """Facebook Connect authentication.
 
@@ -926,9 +941,11 @@ class FacebookMixin(object):
     def _signature(self, args):
         parts = ["%s=%s" % (n, args[n]) for n in sorted(args.keys())]
         body = "".join(parts) + self.settings["facebook_secret"]
-        if isinstance(body, unicode): body = body.encode("utf-8")
+        if isinstance(body, unicode):
+            body = body.encode("utf-8")
         return hashlib.md5(body).hexdigest()
 
+
 class FacebookGraphMixin(OAuth2Mixin):
     """Facebook authentication using the new Graph API and OAuth2."""
     _OAUTH_ACCESS_TOKEN_URL = "https://graph.facebook.com/oauth/access_token?"
@@ -937,68 +954,68 @@ class FacebookGraphMixin(OAuth2Mixin):
 
     def get_authenticated_user(self, redirect_uri, client_id, client_secret,
                               code, callback, extra_fields=None):
-      """Handles the login for the Facebook user, returning a user object.
+        """Handles the login for the Facebook user, returning a user object.
 
-      Example usage::
+        Example usage::
 
-          class FacebookGraphLoginHandler(LoginHandler, tornado.auth.FacebookGraphMixin):
+            class FacebookGraphLoginHandler(LoginHandler, tornado.auth.FacebookGraphMixin):
-            @tornado.web.asynchronous
+              @tornado.web.asynchronous
-            def get(self):
+              def get(self):
-                if self.get_argument("code", False):
+                  if self.get_argument("code", False):
-                    self.get_authenticated_user(
+                      self.get_authenticated_user(
-                      redirect_uri='/auth/facebookgraph/',
+                        redirect_uri='/auth/facebookgraph/',
-                      client_id=self.settings["facebook_api_key"],
+                        client_id=self.settings["facebook_api_key"],
-                      client_secret=self.settings["facebook_secret"],
+                        client_secret=self.settings["facebook_secret"],
-                      code=self.get_argument("code"),
+                        code=self.get_argument("code"),
-                      callback=self.async_callback(
+                        callback=self.async_callback(
-                        self._on_login))
+                          self._on_login))
-                    return
+                      return
-                self.authorize_redirect(redirect_uri='/auth/facebookgraph/',
+                  self.authorize_redirect(redirect_uri='/auth/facebookgraph/',
-                                        client_id=self.settings["facebook_api_key"],
+                                          client_id=self.settings["facebook_api_key"],
-                                        extra_params={"scope": "read_stream,offline_access"})
+                                          extra_params={"scope": "read_stream,offline_access"})
 
-            def _on_login(self, user):
+              def _on_login(self, user):
-              logging.error(user)
+                logging.error(user)
-              self.finish()
+                self.finish()
 
-      """
+        """
-      http = httpclient.AsyncHTTPClient()
+        http = httpclient.AsyncHTTPClient()
-      args = {
+        args = {
-        "redirect_uri": redirect_uri,
+          "redirect_uri": redirect_uri,
-        "code": code,
+          "code": code,
-        "client_id": client_id,
+          "client_id": client_id,
-        "client_secret": client_secret,
+          "client_secret": client_secret,
-      }
+        }
 
-      fields = set(['id', 'name', 'first_name', 'last_name',
+        fields = set(['id', 'name', 'first_name', 'last_name',
-                    'locale', 'picture', 'link'])
+                      'locale', 'picture', 'link'])
-      if extra_fields: fields.update(extra_fields)
+        if extra_fields:
+            fields.update(extra_fields)
 
-      http.fetch(self._oauth_request_token_url(**args),
+        http.fetch(self._oauth_request_token_url(**args),
-          self.async_callback(self._on_access_token, redirect_uri, client_id,
+            self.async_callback(self._on_access_token, redirect_uri, client_id,
-                              client_secret, callback, fields))
+                                client_secret, callback, fields))
 
     def _on_access_token(self, redirect_uri, client_id, client_secret,
                         callback, fields, response):
-      if response.error:
+        if response.error:
-          logging.warning('Facebook auth error: %s' % str(response))
+            logging.warning('Facebook auth error: %s' % str(response))
-          callback(None)
+            callback(None)
-          return
+            return
 
-      args = escape.parse_qs_bytes(escape.native_str(response.body))
+        args = escape.parse_qs_bytes(escape.native_str(response.body))
-      session = {
+        session = {
-          "access_token": args["access_token"][-1],
+            "access_token": args["access_token"][-1],
-          "expires": args.get("expires")
+            "expires": args.get("expires")
-      }
+        }
-
-      self.facebook_request(
-          path="/me",
-          callback=self.async_callback(
-              self._on_get_user_info, callback, session, fields),
-          access_token=session["access_token"],
-          fields=",".join(fields)
-          )
 
+        self.facebook_request(
+            path="/me",
+            callback=self.async_callback(
+                self._on_get_user_info, callback, session, fields),
+            access_token=session["access_token"],
+            fields=",".join(fields)
+            )
 
     def _on_get_user_info(self, callback, session, fields, user):
         if user is None:
@@ -1052,8 +1069,9 @@ class FacebookGraphMixin(OAuth2Mixin):
         if access_token:
             all_args["access_token"] = access_token
             all_args.update(args)
-            all_args.update(post_args or {})
+
-        if all_args: url += "?" + urllib.urlencode(all_args)
+        if all_args:
+            url += "?" + urllib.urlencode(all_args)
         callback = self.async_callback(self._on_facebook_request, callback)
         http = httpclient.AsyncHTTPClient()
         if post_args is not None:
@@ -1070,6 +1088,7 @@ class FacebookGraphMixin(OAuth2Mixin):
             return
         callback(escape.json_decode(response.body))
 
+
 def _oauth_signature(consumer_token, method, url, parameters={}, token=None):
     """Calculates the HMAC-SHA1 OAuth signature for the given request.
 
@@ -1084,7 +1103,7 @@ def _oauth_signature(consumer_token, method, url, parameters={}, token=None):
     base_elems.append(normalized_url)
     base_elems.append("&".join("%s=%s" % (k, _oauth_escape(str(v)))
                                for k, v in sorted(parameters.items())))
-    base_string =  "&".join(_oauth_escape(e) for e in base_elems)
+    base_string = "&".join(_oauth_escape(e) for e in base_elems)
 
     key_elems = [escape.utf8(consumer_token["secret"])]
     key_elems.append(escape.utf8(token["secret"] if token else ""))
@@ -1093,6 +1112,7 @@ def _oauth_signature(consumer_token, method, url, parameters={}, token=None):
     hash = hmac.new(key, escape.utf8(base_string), hashlib.sha1)
     return binascii.b2a_base64(hash.digest())[:-1]
 
+
 def _oauth10a_signature(consumer_token, method, url, parameters={}, token=None):
     """Calculates the HMAC-SHA1 OAuth 1.0a signature for the given request.
 
@@ -1108,7 +1128,7 @@ def _oauth10a_signature(consumer_token, method, url, parameters={}, token=None):
     base_elems.append("&".join("%s=%s" % (k, _oauth_escape(str(v)))
                                for k, v in sorted(parameters.items())))
 
-    base_string =  "&".join(_oauth_escape(e) for e in base_elems)
+    base_string = "&".join(_oauth_escape(e) for e in base_elems)
     key_elems = [escape.utf8(urllib.quote(consumer_token["secret"], safe='~'))]
     key_elems.append(escape.utf8(urllib.quote(token["secret"], safe='~') if token else ""))
     key = b("&").join(key_elems)
@@ -1116,6 +1136,7 @@ def _oauth10a_signature(consumer_token, method, url, parameters={}, token=None):
     hash = hmac.new(key, escape.utf8(base_string), hashlib.sha1)
     return binascii.b2a_base64(hash.digest())[:-1]
 
+
 def _oauth_escape(val):
     if isinstance(val, unicode):
         val = val.encode("utf-8")
@@ -1130,5 +1151,3 @@ def _oauth_parse_response(body):
     special = (b("oauth_token"), b("oauth_token_secret"))
     token.update((k, p[k][0]) for k in p if k not in special)
     return token
-
-

libs/tornado/autoreload.py

@@ -24,9 +24,47 @@ and static resources.
 This module depends on IOLoop, so it will not work in WSGI applications
 and Google AppEngine.  It also will not work correctly when HTTPServer's
 multi-process mode is used.
+
+Reloading loses any Python interpreter command-line arguments (e.g. ``-u``)
+because it re-executes Python using ``sys.executable`` and ``sys.argv``.
+Additionally, modifying these variables will cause reloading to behave
+incorrectly.
 """
 
-from __future__ import with_statement
+from __future__ import absolute_import, division, with_statement
+
+import os
+import sys
+
+# sys.path handling
+# -----------------
+#
+# If a module is run with "python -m", the current directory (i.e. "")
+# is automatically prepended to sys.path, but not if it is run as
+# "path/to/file.py".  The processing for "-m" rewrites the former to
+# the latter, so subsequent executions won't have the same path as the
+# original.
+#
+# Conversely, when run as path/to/file.py, the directory containing
+# file.py gets added to the path, which can cause confusion as imports
+# may become relative in spite of the future import.
+#
+# We address the former problem by setting the $PYTHONPATH environment
+# variable before re-execution so the new process will see the correct
+# path.  We attempt to address the latter problem when tornado.autoreload
+# is run as __main__, although we can't fix the general case because
+# we cannot reliably reconstruct the original command line
+# (http://bugs.python.org/issue14208).
+
+if __name__ == "__main__":
+    # This sys.path manipulation must come before our imports (as much
+    # as possible - if we introduced a tornado.sys or tornado.os
+    # module we'd be in trouble), or else our imports would become
+    # relative again despite the future import.
+    #
+    # There is a separate __main__ block at the end of the file to call main().
+    if sys.path[0] == os.path.dirname(__file__):
+        del sys.path[0]
 
 import functools
 import logging
@@ -44,6 +82,7 @@ try:
 except ImportError:
     signal = None
 
+
 def start(io_loop=None, check_time=500):
     """Restarts the process automatically when a module is modified.
 
@@ -57,6 +96,7 @@ def start(io_loop=None, check_time=500):
     scheduler = ioloop.PeriodicCallback(callback, check_time, io_loop=io_loop)
     scheduler.start()
 
+
 def wait():
     """Wait for a watched file to change, then restart the process.
 
@@ -70,6 +110,7 @@ def wait():
 
 _watched_files = set()
 
+
 def watch(filename):
     """Add a file to the watch list.
 
@@ -79,6 +120,7 @@ def watch(filename):
 
 _reload_hooks = []
 
+
 def add_reload_hook(fn):
     """Add a function to be called before reloading the process.
 
@@ -89,6 +131,7 @@ def add_reload_hook(fn):
     """
     _reload_hooks.append(fn)
 
+
 def _close_all_fds(io_loop):
     for fd in io_loop._handlers.keys():
         try:
@@ -98,6 +141,7 @@ def _close_all_fds(io_loop):
 
 _reload_attempted = False
 
+
 def _reload_on_update(modify_times):
     if _reload_attempted:
         # We already tried to reload and it didn't work, so don't try again.
@@ -112,15 +156,18 @@ def _reload_on_update(modify_times):
         # in the standard library), and occasionally this can cause strange
         # failures in getattr.  Just ignore anything that's not an ordinary
         # module.
-        if not isinstance(module, types.ModuleType): continue
+        if not isinstance(module, types.ModuleType):
+            continue
         path = getattr(module, "__file__", None)
-        if not path: continue
+        if not path:
+            continue
         if path.endswith(".pyc") or path.endswith(".pyo"):
             path = path[:-1]
         _check_file(modify_times, path)
     for path in _watched_files:
         _check_file(modify_times, path)
 
+
 def _check_file(modify_times, path):
     try:
         modified = os.stat(path).st_mtime
@@ -133,6 +180,7 @@ def _check_file(modify_times, path):
         logging.info("%s modified; restarting server", path)
         _reload()
 
+
 def _reload():
     global _reload_attempted
     _reload_attempted = True
@@ -143,6 +191,15 @@ def _reload():
         # ioloop.set_blocking_log_threshold so it doesn't fire
         # after the exec.
         signal.setitimer(signal.ITIMER_REAL, 0, 0)
+    # sys.path fixes: see comments at top of file.  If sys.path[0] is an empty
+    # string, we were (probably) invoked with -m and the effective path
+    # is about to change on re-exec.  Add the current directory to $PYTHONPATH
+    # to ensure that the new process sees the same path we did.
+    path_prefix = '.' + os.pathsep
+    if (sys.path[0] == '' and
+        not os.environ.get("PYTHONPATH", "").startswith(path_prefix)):
+        os.environ["PYTHONPATH"] = (path_prefix +
+                                    os.environ.get("PYTHONPATH", ""))
     if sys.platform == 'win32':
         # os.execv is broken on Windows and can't properly parse command line
         # arguments and executable name if they contain whitespaces. subprocess
@@ -173,6 +230,8 @@ Usage:
   python -m tornado.autoreload -m module.to.run [args...]
   python -m tornado.autoreload path/to/script.py [args...]
 """
+
+
 def main():
     """Command-line wrapper to re-run a script whenever its source changes.
 
@@ -226,25 +285,14 @@ def main():
     if mode == 'module':
         # runpy did a fake import of the module as __main__, but now it's
         # no longer in sys.modules.  Figure out where it is and watch it.
-        watch(pkgutil.get_loader(module).get_filename())
+        loader = pkgutil.get_loader(module)
+        if loader is not None:
+            watch(loader.get_filename())
 
     wait()
 
 
 if __name__ == "__main__":
-    # If this module is run with "python -m tornado.autoreload", the current
+    # See also the other __main__ block at the top of the file, which modifies
-    # directory is automatically prepended to sys.path, but not if it is
+    # sys.path before our imports
-    # run as "path/to/tornado/autoreload.py".  The processing for "-m" rewrites
-    # the former to the latter, so subsequent executions won't have the same
-    # path as the original.  Modify os.environ here to ensure that the
-    # re-executed process will have the same path.
-    # Conversely, when run as path/to/tornado/autoreload.py, the directory
-    # containing autoreload.py gets added to the path, but we don't want
-    # tornado modules importable at top level, so remove it.
-    path_prefix = '.' + os.pathsep
-    if (sys.path[0] == '' and
-        not os.environ.get("PYTHONPATH", "").startswith(path_prefix)):
-        os.environ["PYTHONPATH"] = path_prefix + os.environ.get("PYTHONPATH", "")
-    elif sys.path[0] == os.path.dirname(__file__):
-        del sys.path[0]
     main()

libs/tornado/curl_httpclient.py

@@ -16,7 +16,7 @@
 
 """Blocking and non-blocking HTTP client implementations using pycurl."""
 
-from __future__ import with_statement
+from __future__ import absolute_import, division, with_statement
 
 import cStringIO
 import collections
@@ -32,6 +32,7 @@ from tornado import stack_context
 from tornado.escape import utf8
 from tornado.httpclient import HTTPRequest, HTTPResponse, HTTPError, AsyncHTTPClient, main
 
+
 class CurlAsyncHTTPClient(AsyncHTTPClient):
     def initialize(self, io_loop=None, max_clients=10,
                    max_simultaneous_connections=None):
@@ -109,15 +110,17 @@ class CurlAsyncHTTPClient(AsyncHTTPClient):
         if self._timeout is not None:
             self.io_loop.remove_timeout(self._timeout)
         self._timeout = self.io_loop.add_timeout(
-            time.time() + msecs/1000.0, self._handle_timeout)
+            time.time() + msecs / 1000.0, self._handle_timeout)
 
     def _handle_events(self, fd, events):
         """Called by IOLoop when there is activity on one of our
         file descriptors.
         """
         action = 0
-        if events & ioloop.IOLoop.READ: action |= pycurl.CSELECT_IN
+        if events & ioloop.IOLoop.READ:
-        if events & ioloop.IOLoop.WRITE: action |= pycurl.CSELECT_OUT
+            action |= pycurl.CSELECT_IN
+        if events & ioloop.IOLoop.WRITE:
+            action |= pycurl.CSELECT_OUT
         while True:
             try:
                 ret, num_handles = self._socket_action(fd, action)
@@ -250,7 +253,6 @@ class CurlAsyncHTTPClient(AsyncHTTPClient):
         except Exception:
             self.handle_callback_exception(info["callback"])
 
-
     def handle_callback_exception(self, callback):
         self.io_loop.handle_callback_exception(callback)
 
@@ -372,7 +374,7 @@ def _curl_setup_request(curl, request, buffer, headers):
 
     # Handle curl's cryptic options for every individual HTTP method
     if request.method in ("POST", "PUT"):
-        request_buffer =  cStringIO.StringIO(utf8(request.body))
+        request_buffer = cStringIO.StringIO(utf8(request.body))
         curl.setopt(pycurl.READFUNCTION, request_buffer.read)
         if request.method == "POST":
             def ioctl(cmd):
@@ -393,8 +395,11 @@ def _curl_setup_request(curl, request, buffer, headers):
         curl.unsetopt(pycurl.USERPWD)
         logging.debug("%s %s", request.method, request.url)
 
-    if request.client_key is not None or request.client_cert is not None:
+    if request.client_cert is not None:
-        raise ValueError("Client certificate not supported with curl_httpclient")
+        curl.setopt(pycurl.SSLCERT, request.client_cert)
+
+    if request.client_key is not None:
+        curl.setopt(pycurl.SSLKEY, request.client_key)
 
     if threading.activeCount() > 1:
         # libcurl/pycurl is not thread-safe by default.  When multiple threads
@@ -420,6 +425,7 @@ def _curl_header_callback(headers, header_line):
         return
     headers.parse_line(header_line)
 
+
 def _curl_debug(debug_type, debug_msg):
     debug_types = ('I', '<', '>', '<', '>')
     if debug_type == 0:

libs/tornado/database.py

@@ -16,14 +16,24 @@
 
 """A lightweight wrapper around MySQLdb."""
 
+from __future__ import absolute_import, division, with_statement
+
 import copy
-import MySQLdb.constants
-import MySQLdb.converters
-import MySQLdb.cursors
 import itertools
 import logging
 import time
 
+try:
+    import MySQLdb.constants
+    import MySQLdb.converters
+    import MySQLdb.cursors
+except ImportError:
+    # If MySQLdb isn't available this module won't actually be useable,
+    # but we want it to at least be importable (mainly for readthedocs.org,
+    # which has limitations on third-party modules)
+    MySQLdb = None
+
+
 class Connection(object):
     """A lightweight wrapper around MySQLdb DB-API connections.
 
@@ -41,7 +51,7 @@ class Connection(object):
     UTF-8 on all connections to avoid time zone and encoding errors.
     """
     def __init__(self, host, database, user=None, password=None,
-                 max_idle_time=7*3600):
+                 max_idle_time=7 * 3600):
         self.host = host
         self.database = database
         self.max_idle_time = max_idle_time
@@ -210,20 +220,19 @@ class Row(dict):
         except KeyError:
             raise AttributeError(name)
 
+if MySQLdb is not None:
+    # Fix the access conversions to properly recognize unicode/binary
+    FIELD_TYPE = MySQLdb.constants.FIELD_TYPE
+    FLAG = MySQLdb.constants.FLAG
+    CONVERSIONS = copy.copy(MySQLdb.converters.conversions)
 
-# Fix the access conversions to properly recognize unicode/binary
+    field_types = [FIELD_TYPE.BLOB, FIELD_TYPE.STRING, FIELD_TYPE.VAR_STRING]
-FIELD_TYPE = MySQLdb.constants.FIELD_TYPE
+    if 'VARCHAR' in vars(FIELD_TYPE):
-FLAG = MySQLdb.constants.FLAG
+        field_types.append(FIELD_TYPE.VARCHAR)
-CONVERSIONS = copy.copy(MySQLdb.converters.conversions)
 
-field_types = [FIELD_TYPE.BLOB, FIELD_TYPE.STRING, FIELD_TYPE.VAR_STRING]
+    for field_type in field_types:
-if 'VARCHAR' in vars(FIELD_TYPE):
+        CONVERSIONS[field_type] = [(FLAG.BINARY, str)] + CONVERSIONS[field_type]
-    field_types.append(FIELD_TYPE.VARCHAR)
 
-for field_type in field_types:
+    # Alias some common MySQL exceptions
-    CONVERSIONS[field_type] = [(FLAG.BINARY, str)] + CONVERSIONS[field_type]
+    IntegrityError = MySQLdb.IntegrityError
-
+    OperationalError = MySQLdb.OperationalError
-
-# Alias some common MySQL exceptions
-IntegrityError = MySQLdb.IntegrityError
-OperationalError = MySQLdb.OperationalError

libs/tornado/escape.py

@@ -20,14 +20,18 @@ Also includes a few other miscellaneous string manipulation functions that
 have crept in over time.
 """
 
+from __future__ import absolute_import, division, with_statement
+
 import htmlentitydefs
 import re
 import sys
 import urllib
 
 # Python3 compatibility:  On python2.5, introduce the bytes alias from 2.6
-try: bytes
+try:
-except Exception: bytes = str
+    bytes
+except Exception:
+    bytes = str
 
 try:
     from urlparse import parse_qs  # Python 2.6+
@@ -62,6 +66,8 @@ except Exception:
 
 _XHTML_ESCAPE_RE = re.compile('[&<>"]')
 _XHTML_ESCAPE_DICT = {'&': '&amp;', '<': '&lt;', '>': '&gt;', '"': '&quot;'}
+
+
 def xhtml_escape(value):
     """Escapes a string so it is valid within XML or XHTML."""
     return _XHTML_ESCAPE_RE.sub(lambda match: _XHTML_ESCAPE_DICT[match.group(0)],
@@ -143,13 +149,14 @@ else:
         result = parse_qs(qs, keep_blank_values, strict_parsing,
                           encoding='latin1', errors='strict')
         encoded = {}
-        for k,v in result.iteritems():
+        for k, v in result.iteritems():
             encoded[k] = [i.encode('latin1') for i in v]
         return encoded
 
 
-
 _UTF8_TYPES = (bytes, type(None))
+
+
 def utf8(value):
     """Converts a string argument to a byte string.
 
@@ -162,6 +169,8 @@ def utf8(value):
     return value.encode("utf-8")
 
 _TO_UNICODE_TYPES = (unicode, type(None))
+
+
 def to_unicode(value):
     """Converts a string argument to a unicode string.
 
@@ -185,6 +194,8 @@ else:
     native_str = utf8
 
 _BASESTRING_TYPES = (basestring, type(None))
+
+
 def to_basestring(value):
     """Converts a string argument to a subclass of basestring.
 
@@ -199,13 +210,14 @@ def to_basestring(value):
     assert isinstance(value, bytes)
     return value.decode("utf-8")
 
+
 def recursive_unicode(obj):
     """Walks a simple data structure, converting byte strings to unicode.
 
     Supports lists, tuples, and dictionaries.
     """
     if isinstance(obj, dict):
-        return dict((recursive_unicode(k), recursive_unicode(v)) for (k,v) in obj.iteritems())
+        return dict((recursive_unicode(k), recursive_unicode(v)) for (k, v) in obj.iteritems())
     elif isinstance(obj, list):
         return list(recursive_unicode(i) for i in obj)
     elif isinstance(obj, tuple):
@@ -234,8 +246,17 @@ def linkify(text, shorten=False, extra_params="",
 
     shorten: Long urls will be shortened for display.
 
-    extra_params: Extra text to include in the link tag,
+    extra_params: Extra text to include in the link tag, or a callable
-        e.g. linkify(text, extra_params='rel="nofollow" class="external"')
+        taking the link as an argument and returning the extra text
+        e.g. ``linkify(text, extra_params='rel="nofollow" class="external"')``,
+        or::
+
+            def extra_params_cb(url):
+                if url.startswith("http://example.com"):
+                    return 'class="internal"'
+                else:
+                    return 'class="external" rel="nofollow"'
+            linkify(text, extra_params=extra_params_cb)
 
     require_protocol: Only linkify urls which include a protocol. If this is
         False, urls such as www.facebook.com will also be linkified.
@@ -244,7 +265,7 @@ def linkify(text, shorten=False, extra_params="",
         e.g. linkify(text, permitted_protocols=["http", "ftp", "mailto"]).
         It is very unsafe to include protocols such as "javascript".
     """
-    if extra_params:
+    if extra_params and not callable(extra_params):
         extra_params = " " + extra_params.strip()
 
     def make_link(m):
@@ -260,7 +281,10 @@ def linkify(text, shorten=False, extra_params="",
         if not proto:
             href = "http://" + href   # no proto specified, use http
 
-        params = extra_params
+        if callable(extra_params):
+            params = " " + extra_params(href).strip()
+        else:
+            params = extra_params
 
         # clip long urls. max_len is just an approximation
         max_len = 30

libs/tornado/gen.py

@@ -62,7 +62,7 @@ it was called with one argument, the result is that argument.  If it was
 called with more than one argument or any keyword arguments, the result
 is an `Arguments` object, which is a named tuple ``(args, kwargs)``.
 """
-from __future__ import with_statement
+from __future__ import absolute_import, division, with_statement
 
 import functools
 import operator
@@ -71,10 +71,22 @@ import types
 
 from tornado.stack_context import ExceptionStackContext
 
-class KeyReuseError(Exception): pass
+
-class UnknownKeyError(Exception): pass
+class KeyReuseError(Exception):
-class LeakedCallbackError(Exception): pass
+    pass
-class BadYieldError(Exception): pass
+
+
+class UnknownKeyError(Exception):
+    pass
+
+
+class LeakedCallbackError(Exception):
+    pass
+
+
+class BadYieldError(Exception):
+    pass
+
 
 def engine(func):
     """Decorator for asynchronous generators.
@@ -92,6 +104,7 @@ def engine(func):
     @functools.wraps(func)
     def wrapper(*args, **kwargs):
         runner = None
+
         def handle_exception(typ, value, tb):
             # if the function throws an exception before its first "yield"
             # (or is not a generator at all), the Runner won't exist yet.
@@ -100,16 +113,18 @@ def engine(func):
             if runner is not None:
                 return runner.handle_exception(typ, value, tb)
             return False
-        with ExceptionStackContext(handle_exception):
+        with ExceptionStackContext(handle_exception) as deactivate:
             gen = func(*args, **kwargs)
             if isinstance(gen, types.GeneratorType):
-                runner = Runner(gen)
+                runner = Runner(gen, deactivate)
                 runner.run()
                 return
             assert gen is None, gen
+            deactivate()
             # no yield, so we're done
     return wrapper
 
+
 class YieldPoint(object):
     """Base class for objects that may be yielded from the generator."""
     def start(self, runner):
@@ -134,6 +149,7 @@ class YieldPoint(object):
         """
         raise NotImplementedError()
 
+
 class Callback(YieldPoint):
     """Returns a callable object that will allow a matching `Wait` to proceed.
 
@@ -159,6 +175,7 @@ class Callback(YieldPoint):
     def get_result(self):
         return self.runner.result_callback(self.key)
 
+
 class Wait(YieldPoint):
     """Returns the argument passed to the result of a previous `Callback`."""
     def __init__(self, key):
@@ -173,6 +190,7 @@ class Wait(YieldPoint):
     def get_result(self):
         return self.runner.pop_result(self.key)
 
+
 class WaitAll(YieldPoint):
     """Returns the results of multiple previous `Callbacks`.
 
@@ -228,6 +246,7 @@ class Task(YieldPoint):
     def get_result(self):
         return self.runner.pop_result(self.key)
 
+
 class Multi(YieldPoint):
     """Runs multiple asynchronous operations in parallel.
 
@@ -250,21 +269,26 @@ class Multi(YieldPoint):
     def get_result(self):
         return [i.get_result() for i in self.children]
 
+
 class _NullYieldPoint(YieldPoint):
     def start(self, runner):
         pass
+
     def is_ready(self):
         return True
+
     def get_result(self):
         return None
 
+
 class Runner(object):
     """Internal implementation of `tornado.gen.engine`.
 
     Maintains information about pending callbacks and their results.
     """
-    def __init__(self, gen):
+    def __init__(self, gen, deactivate_stack_context):
         self.gen = gen
+        self.deactivate_stack_context = deactivate_stack_context
         self.yield_point = _NullYieldPoint()
         self.pending_callbacks = set()
         self.results = {}
@@ -329,6 +353,7 @@ class Runner(object):
                         raise LeakedCallbackError(
                             "finished without waiting for callbacks %r" %
                             self.pending_callbacks)
+                    self.deactivate_stack_context()
                     return
                 except Exception:
                     self.finished = True
@@ -366,6 +391,8 @@ class Runner(object):
             return False
 
 # in python 2.6+ this could be a collections.namedtuple
+
+
 class Arguments(tuple):
     """The result of a yield expression whose callback had more than one
     argument (or keyword arguments).

libs/tornado/httpclient.py

@@ -29,6 +29,8 @@ you use a recent version of ``libcurl`` and ``pycurl``.  Currently the minimum
 supported version is 7.18.2, and the recommended version is 7.21.1 or newer.
 """
 
+from __future__ import absolute_import, division, with_statement
+
 import calendar
 import email.utils
 import httplib
@@ -40,6 +42,7 @@ from tornado import httputil
 from tornado.ioloop import IOLoop
 from tornado.util import import_object, bytes_type
 
+
 class HTTPClient(object):
     """A blocking HTTP client.
 
@@ -54,11 +57,11 @@ class HTTPClient(object):
         except httpclient.HTTPError, e:
             print "Error:", e
     """
-    def __init__(self, async_client_class=None):
+    def __init__(self, async_client_class=None, **kwargs):
         self._io_loop = IOLoop()
         if async_client_class is None:
             async_client_class = AsyncHTTPClient
-        self._async_client = async_client_class(self._io_loop)
+        self._async_client = async_client_class(self._io_loop, **kwargs)
         self._response = None
         self._closed = False
 
@@ -91,6 +94,7 @@ class HTTPClient(object):
         response.rethrow()
         return response
 
+
 class AsyncHTTPClient(object):
     """An non-blocking HTTP client.
 
@@ -120,6 +124,8 @@ class AsyncHTTPClient(object):
     _impl_class = None
     _impl_kwargs = None
 
+    _DEFAULT_MAX_CLIENTS = 10
+
     @classmethod
     def _async_clients(cls):
         assert cls is not AsyncHTTPClient, "should only be called on subclasses"
@@ -127,7 +133,7 @@ class AsyncHTTPClient(object):
             cls._async_client_dict = weakref.WeakKeyDictionary()
         return cls._async_client_dict
 
-    def __new__(cls, io_loop=None, max_clients=10, force_instance=False, 
+    def __new__(cls, io_loop=None, max_clients=None, force_instance=False,
                 **kwargs):
         io_loop = io_loop or IOLoop.instance()
         if cls is AsyncHTTPClient:
@@ -145,7 +151,13 @@ class AsyncHTTPClient(object):
             if cls._impl_kwargs:
                 args.update(cls._impl_kwargs)
             args.update(kwargs)
-            instance.initialize(io_loop, max_clients, **args)
+            if max_clients is not None:
+                # max_clients is special because it may be passed
+                # positionally instead of by keyword
+                args["max_clients"] = max_clients
+            elif "max_clients" not in args:
+                args["max_clients"] = AsyncHTTPClient._DEFAULT_MAX_CLIENTS
+            instance.initialize(io_loop, **args)
             if not force_instance:
                 impl._async_clients()[io_loop] = instance
             return instance
@@ -200,6 +212,16 @@ class AsyncHTTPClient(object):
         AsyncHTTPClient._impl_class = impl
         AsyncHTTPClient._impl_kwargs = kwargs
 
+    @staticmethod
+    def _save_configuration():
+        return (AsyncHTTPClient._impl_class, AsyncHTTPClient._impl_kwargs)
+
+    @staticmethod
+    def _restore_configuration(saved):
+        AsyncHTTPClient._impl_class = saved[0]
+        AsyncHTTPClient._impl_kwargs = saved[1]
+
+
 class HTTPRequest(object):
     """HTTP client request object."""
     def __init__(self, url, method="GET", headers=None, body=None,
@@ -325,12 +347,15 @@ class HTTPResponse(object):
         plus 'queue', which is the delay (if any) introduced by waiting for
         a slot under AsyncHTTPClient's max_clients setting.
     """
-    def __init__(self, request, code, headers={}, buffer=None,
+    def __init__(self, request, code, headers=None, buffer=None,
                  effective_url=None, error=None, request_time=None,
-                 time_info={}):
+                 time_info=None):
         self.request = request
         self.code = code
-        self.headers = headers
+        if headers is not None:
+            self.headers = headers
+        else:
+            self.headers = httputil.HTTPHeaders()
         self.buffer = buffer
         self._body = None
         if effective_url is None:
@@ -345,7 +370,7 @@ class HTTPResponse(object):
         else:
             self.error = error
         self.request_time = request_time
-        self.time_info = time_info
+        self.time_info = time_info or {}
 
     def _get_body(self):
         if self.buffer is None:

libs/tornado/httpserver.py

@@ -24,13 +24,14 @@ This module also defines the `HTTPRequest` class which is exposed via
 `tornado.web.RequestHandler.request`.
 """
 
+from __future__ import absolute_import, division, with_statement
+
 import Cookie
 import logging
 import socket
 import time
-import urlparse
 
-from tornado.escape import utf8, native_str, parse_qs_bytes
+from tornado.escape import native_str, parse_qs_bytes
 from tornado import httputil
 from tornado import iostream
 from tornado.netutil import TCPServer
@@ -38,10 +39,11 @@ from tornado import stack_context
 from tornado.util import b, bytes_type
 
 try:
-    import ssl # Python 2.6+
+    import ssl  # Python 2.6+
 except ImportError:
     ssl = None
 
+
 class HTTPServer(TCPServer):
     r"""A non-blocking, single-threaded HTTP server.
 
@@ -143,10 +145,12 @@ class HTTPServer(TCPServer):
         HTTPConnection(stream, address, self.request_callback,
                        self.no_keep_alive, self.xheaders)
 
+
 class _BadRequestException(Exception):
     """Exception class for malformed HTTP requests."""
     pass
 
+
 class HTTPConnection(object):
     """Handles a connection to an HTTP client, executing HTTP requests.
 
@@ -156,9 +160,6 @@ class HTTPConnection(object):
     def __init__(self, stream, address, request_callback, no_keep_alive=False,
                  xheaders=False):
         self.stream = stream
-        if self.stream.socket.family not in (socket.AF_INET, socket.AF_INET6):
-            # Unix (or other) socket; fake the remote address
-            address = ('0.0.0.0', 0)
         self.address = address
         self.request_callback = request_callback
         self.no_keep_alive = no_keep_alive
@@ -233,9 +234,20 @@ class HTTPConnection(object):
             if not version.startswith("HTTP/"):
                 raise _BadRequestException("Malformed HTTP version in HTTP Request-Line")
             headers = httputil.HTTPHeaders.parse(data[eol:])
+
+            # HTTPRequest wants an IP, not a full socket address
+            if getattr(self.stream.socket, 'family', socket.AF_INET) in (
+                socket.AF_INET, socket.AF_INET6):
+                # Jython 2.5.2 doesn't have the socket.family attribute,
+                # so just assume IP in that case.
+                remote_ip = self.address[0]
+            else:
+                # Unix (or other) socket; fake the remote address
+                remote_ip = '0.0.0.0'
+
             self._request = HTTPRequest(
                 connection=self, method=method, uri=uri, version=version,
-                headers=headers, remote_ip=self.address[0])
+                headers=headers, remote_ip=remote_ip)
 
             content_length = headers.get("Content-Length")
             if content_length:
@@ -256,27 +268,10 @@ class HTTPConnection(object):
 
     def _on_request_body(self, data):
         self._request.body = data
-        content_type = self._request.headers.get("Content-Type", "")
+        if self._request.method in ("POST", "PATCH", "PUT"):
-        if self._request.method in ("POST", "PUT"):
+            httputil.parse_body_arguments(
-            if content_type.startswith("application/x-www-form-urlencoded"):
+                self._request.headers.get("Content-Type", ""), data,
-                arguments = parse_qs_bytes(native_str(self._request.body))
+                self._request.arguments, self._request.files)
-                for name, values in arguments.iteritems():
-                    values = [v for v in values if v]
-                    if values:
-                        self._request.arguments.setdefault(name, []).extend(
-                            values)
-            elif content_type.startswith("multipart/form-data"):
-                fields = content_type.split(";")
-                for field in fields:
-                    k, sep, v = field.strip().partition("=")
-                    if k == "boundary" and v:
-                        httputil.parse_multipart_form_data(
-                            utf8(v), data,
-                            self._request.arguments,
-                            self._request.files)
-                        break
-                else:
-                    logging.warning("Invalid multipart/form-data")
         self.request_callback(self._request)
 
 
@@ -386,14 +381,13 @@ class HTTPRequest(object):
         self._start_time = time.time()
         self._finish_time = None
 
-        scheme, netloc, path, query, fragment = urlparse.urlsplit(native_str(uri))
+        self.path, sep, self.query = uri.partition('?')
-        self.path = path
+        arguments = parse_qs_bytes(self.query)
-        self.query = query
-        arguments = parse_qs_bytes(query)
         self.arguments = {}
         for name, values in arguments.iteritems():
             values = [v for v in values if v]
-            if values: self.arguments[name] = values
+            if values:
+                self.arguments[name] = values
 
     def supports_http_1_1(self):
         """Returns True if this request supports HTTP/1.1 semantics"""
@@ -473,4 +467,3 @@ class HTTPRequest(object):
                 return False
             raise
         return True
-

libs/tornado/httputil.py

@@ -16,12 +16,16 @@
 
 """HTTP utility code shared by clients and servers."""
 
+from __future__ import absolute_import, division, with_statement
+
 import logging
 import urllib
 import re
 
+from tornado.escape import native_str, parse_qs_bytes, utf8
 from tornado.util import b, ObjectDict
 
+
 class HTTPHeaders(dict):
     """A dictionary that maintains Http-Header-Case for all keys.
 
@@ -55,7 +59,14 @@ class HTTPHeaders(dict):
         dict.__init__(self)
         self._as_list = {}
         self._last_key = None
-        self.update(*args, **kwargs)
+        if (len(args) == 1 and len(kwargs) == 0 and
+            isinstance(args[0], HTTPHeaders)):
+            # Copy constructor
+            for k, v in args[0].get_all():
+                self.add(k, v)
+        else:
+            # Dict-style initialization
+            self.update(*args, **kwargs)
 
     # new public methods
 
@@ -144,6 +155,10 @@ class HTTPHeaders(dict):
         for k, v in dict(*args, **kwargs).iteritems():
             self[k] = v
 
+    def copy(self):
+        # default implementation returns dict(self), not the subclass
+        return HTTPHeaders(self)
+
     _NORMALIZED_HEADER_RE = re.compile(r'^[A-Z0-9][a-z0-9]*(-[A-Z0-9][a-z0-9]*)*$')
     _normalized_headers = {}
 
@@ -172,7 +187,8 @@ def url_concat(url, args):
     >>> url_concat("http://example.com/foo?a=b", dict(c="d"))
     'http://example.com/foo?a=b&c=d'
     """
-    if not args: return url
+    if not args:
+        return url
     if url[-1] not in ('?', '&'):
         url += '&' if ('?' in url) else '?'
     return url + urllib.urlencode(args)
@@ -190,6 +206,24 @@ class HTTPFile(ObjectDict):
     pass
 
 
+def parse_body_arguments(content_type, body, arguments, files):
+    if content_type.startswith("application/x-www-form-urlencoded"):
+        uri_arguments = parse_qs_bytes(native_str(body))
+        for name, values in uri_arguments.iteritems():
+            values = [v for v in values if v]
+            if values:
+                arguments.setdefault(name, []).extend(values)
+    elif content_type.startswith("multipart/form-data"):
+        fields = content_type.split(";")
+        for field in fields:
+            k, sep, v = field.strip().partition("=")
+            if k == "boundary" and v:
+                parse_multipart_form_data(utf8(v), body, arguments, files)
+                break
+        else:
+            logging.warning("Invalid multipart/form-data")
+
+
 def parse_multipart_form_data(boundary, data, arguments, files):
     """Parses a multipart/form-data body.
 
@@ -204,13 +238,14 @@ def parse_multipart_form_data(boundary, data, arguments, files):
     # in the wild.
     if boundary.startswith(b('"')) and boundary.endswith(b('"')):
         boundary = boundary[1:-1]
-    if data.endswith(b("\r\n")):
+    final_boundary_index = data.rfind(b("--") + boundary + b("--"))
-        footer_length = len(boundary) + 6
+    if final_boundary_index == -1:
-    else:
+        logging.warning("Invalid multipart/form-data: no final boundary")
-        footer_length = len(boundary) + 4
+        return
-    parts = data[:-footer_length].split(b("--") + boundary + b("\r\n"))
+    parts = data[:final_boundary_index].split(b("--") + boundary + b("\r\n"))
     for part in parts:
-        if not part: continue
+        if not part:
+            continue
         eoh = part.find(b("\r\n\r\n"))
         if eoh == -1:
             logging.warning("multipart/form-data missing headers")
@@ -250,6 +285,7 @@ def _parseparam(s):
         yield f.strip()
         s = s[end:]
 
+
 def _parse_header(line):
     """Parse a Content-type like header.
 
@@ -263,7 +299,7 @@ def _parse_header(line):
         i = p.find('=')
         if i >= 0:
             name = p[:i].strip().lower()
-            value = p[i+1:].strip()
+            value = p[i + 1:].strip()
             if len(value) >= 2 and value[0] == value[-1] == '"':
                 value = value[1:-1]
                 value = value.replace('\\\\', '\\').replace('\\"', '"')
@@ -274,7 +310,3 @@ def _parse_header(line):
 def doctests():
     import doctest
     return doctest.DocTestSuite()
-
-if __name__ == "__main__":
-    import doctest
-    doctest.testmod()

libs/tornado/ioloop.py

@@ -26,7 +26,7 @@ In addition to I/O events, the `IOLoop` can also schedule time-based events.
 `IOLoop.add_timeout` is a non-blocking alternative to `time.sleep`.
 """
 
-from __future__ import with_statement
+from __future__ import absolute_import, division, with_statement
 
 import datetime
 import errno
@@ -104,6 +104,9 @@ class IOLoop(object):
     WRITE = _EPOLLOUT
     ERROR = _EPOLLERR | _EPOLLHUP
 
+    # Global lock for creating global IOLoop instance
+    _instance_lock = threading.Lock()
+
     def __init__(self, impl=None):
         self._impl = impl or _poll()
         if hasattr(self._impl, 'fileno'):
@@ -142,7 +145,10 @@ class IOLoop(object):
                     self.io_loop = io_loop or IOLoop.instance()
         """
         if not hasattr(IOLoop, "_instance"):
-            IOLoop._instance = IOLoop()
+            with IOLoop._instance_lock:
+                if not hasattr(IOLoop, "_instance"):
+                    # New instance after double check
+                    IOLoop._instance = IOLoop()
         return IOLoop._instance
 
     @staticmethod
@@ -164,7 +170,20 @@ class IOLoop(object):
         """Closes the IOLoop, freeing any resources used.
 
         If ``all_fds`` is true, all file descriptors registered on the
-        IOLoop will be closed (not just the ones created by the IOLoop itself.
+        IOLoop will be closed (not just the ones created by the IOLoop itself).
+
+        Many applications will only use a single IOLoop that runs for the
+        entire lifetime of the process.  In that case closing the IOLoop
+        is not necessary since everything will be cleaned up when the
+        process exits.  `IOLoop.close` is provided mainly for scenarios
+        such as unit tests, which create and destroy a large number of
+        IOLoops.
+
+        An IOLoop must be completely stopped before it can be closed.  This
+        means that `IOLoop.stop()` must be called *and* `IOLoop.start()` must
+        be allowed to return before attempting to call `IOLoop.close()`.
+        Therefore the call to `close` will usually appear just after
+        the call to `start` rather than near the call to `stop`.
         """
         self.remove_handler(self._waker.fileno())
         if all_fds:
@@ -335,6 +354,9 @@ class IOLoop(object):
 
         ioloop.start() will return after async_method has run its callback,
         whether that callback was invoked before or after ioloop.start.
+
+        Note that even after `stop` has been called, the IOLoop is not
+        completely stopped until `IOLoop.start` has also returned.
         """
         self._running = False
         self._stopped = True
@@ -431,7 +453,7 @@ class _Timeout(object):
     @staticmethod
     def timedelta_to_seconds(td):
         """Equivalent to td.total_seconds() (introduced in python 2.7)."""
-        return (td.microseconds + (td.seconds + td.days * 24 * 3600) * 10**6) / float(10**6)
+        return (td.microseconds + (td.seconds + td.days * 24 * 3600) * 10 ** 6) / float(10 ** 6)
 
     # Comparison methods to sort by deadline, with object id as a tiebreaker
     # to guarantee a consistent ordering.  The heapq module uses __le__
@@ -474,7 +496,8 @@ class PeriodicCallback(object):
             self._timeout = None
 
     def _run(self):
-        if not self._running: return
+        if not self._running:
+            return
         try:
             self.callback()
         except Exception:
@@ -530,6 +553,8 @@ class _KQueue(object):
         self._kqueue.close()
 
     def register(self, fd, events):
+        if fd in self._active:
+            raise IOError("fd %d already registered" % fd)
         self._control(fd, events, select.KQ_EV_ADD)
         self._active[fd] = events
 
@@ -591,8 +616,12 @@ class _Select(object):
         pass
 
     def register(self, fd, events):
-        if events & IOLoop.READ: self.read_fds.add(fd)
+        if fd in self.read_fds or fd in self.write_fds or fd in self.error_fds:
-        if events & IOLoop.WRITE: self.write_fds.add(fd)
+            raise IOError("fd %d already registered" % fd)
+        if events & IOLoop.READ:
+            self.read_fds.add(fd)
+        if events & IOLoop.WRITE:
+            self.write_fds.add(fd)
         if events & IOLoop.ERROR:
             self.error_fds.add(fd)
             # Closed connections are reported as errors by epoll and kqueue,
@@ -633,7 +662,7 @@ elif hasattr(select, "kqueue"):
 else:
     try:
         # Linux systems with our C module installed
-        import epoll
+        from tornado import epoll
         _poll = _EPoll
     except Exception:
         # All other systems

libs/tornado/iostream.py

@@ -16,11 +16,12 @@
 
 """A utility class to write to and read from a non-blocking socket."""
 
-from __future__ import with_statement
+from __future__ import absolute_import, division, with_statement
 
 import collections
 import errno
 import logging
+import os
 import socket
 import sys
 import re
@@ -30,10 +31,11 @@ from tornado import stack_context
 from tornado.util import b, bytes_type
 
 try:
-    import ssl # Python 2.6+
+    import ssl  # Python 2.6+
 except ImportError:
     ssl = None
 
+
 class IOStream(object):
     r"""A utility class to write to and read from a non-blocking socket.
 
@@ -47,6 +49,9 @@ class IOStream(object):
     and may either be connected before passing it to the IOStream or
     connected with IOStream.connect.
 
+    When a stream is closed due to an error, the IOStream's `error`
+    attribute contains the exception object.
+
     A very simple (and broken) HTTP client using this class::
 
         from tornado import ioloop
@@ -83,6 +88,7 @@ class IOStream(object):
         self.io_loop = io_loop or ioloop.IOLoop.instance()
         self.max_buffer_size = max_buffer_size
         self.read_chunk_size = read_chunk_size
+        self.error = None
         self._read_buffer = collections.deque()
         self._write_buffer = collections.deque()
         self._read_buffer_size = 0
@@ -136,31 +142,15 @@ class IOStream(object):
 
     def read_until_regex(self, regex, callback):
         """Call callback when we read the given regex pattern."""
-        assert not self._read_callback, "Already reading"
+        self._set_read_callback(callback)
         self._read_regex = re.compile(regex)
-        self._read_callback = stack_context.wrap(callback)
+        self._try_inline_read()
-        while True:
-            # See if we've already got the data from a previous read
-            if self._read_from_buffer():
-                return
-            self._check_closed()
-            if self._read_to_buffer() == 0:
-                break
-        self._add_io_state(self.io_loop.READ)
 
     def read_until(self, delimiter, callback):
         """Call callback when we read the given delimiter."""
-        assert not self._read_callback, "Already reading"
+        self._set_read_callback(callback)
         self._read_delimiter = delimiter
-        self._read_callback = stack_context.wrap(callback)
+        self._try_inline_read()
-        while True:
-            # See if we've already got the data from a previous read
-            if self._read_from_buffer():
-                return
-            self._check_closed()
-            if self._read_to_buffer() == 0:
-                break
-        self._add_io_state(self.io_loop.READ)
 
     def read_bytes(self, num_bytes, callback, streaming_callback=None):
         """Call callback when we read the given number of bytes.
@@ -169,18 +159,11 @@ class IOStream(object):
         of data as they become available, and the argument to the final
         ``callback`` will be empty.
         """
-        assert not self._read_callback, "Already reading"
+        self._set_read_callback(callback)
         assert isinstance(num_bytes, (int, long))
         self._read_bytes = num_bytes
-        self._read_callback = stack_context.wrap(callback)
         self._streaming_callback = stack_context.wrap(streaming_callback)
-        while True:
+        self._try_inline_read()
-            if self._read_from_buffer():
-                return
-            self._check_closed()
-            if self._read_to_buffer() == 0:
-                break
-        self._add_io_state(self.io_loop.READ)
 
     def read_until_close(self, callback, streaming_callback=None):
         """Reads all data from the socket until it is closed.
@@ -192,12 +175,12 @@ class IOStream(object):
         Subject to ``max_buffer_size`` limit from `IOStream` constructor if
         a ``streaming_callback`` is not used.
         """
-        assert not self._read_callback, "Already reading"
+        self._set_read_callback(callback)
         if self.closed():
             self._run_callback(callback, self._consume(self._read_buffer_size))
+            self._read_callback = None
             return
         self._read_until_close = True
-        self._read_callback = stack_context.wrap(callback)
         self._streaming_callback = stack_context.wrap(streaming_callback)
         self._add_io_state(self.io_loop.READ)
 
@@ -211,10 +194,18 @@ class IOStream(object):
         """
         assert isinstance(data, bytes_type)
         self._check_closed()
+        # We use bool(_write_buffer) as a proxy for write_buffer_size>0,
+        # so never put empty strings in the buffer.
         if data:
-            # We use bool(_write_buffer) as a proxy for write_buffer_size>0,
+            # Break up large contiguous strings before inserting them in the
-            # so never put empty strings in the buffer.
+            # write buffer, so we don't have to recopy the entire thing
-            self._write_buffer.append(data)
+            # as we slice off pieces to send to the socket.
+            WRITE_BUFFER_CHUNK_SIZE = 128 * 1024
+            if len(data) > WRITE_BUFFER_CHUNK_SIZE:
+                for i in range(0, len(data), WRITE_BUFFER_CHUNK_SIZE):
+                    self._write_buffer.append(data[i:i + WRITE_BUFFER_CHUNK_SIZE])
+            else:
+                self._write_buffer.append(data)
         self._write_callback = stack_context.wrap(callback)
         self._handle_write()
         if self._write_buffer:
@@ -228,6 +219,8 @@ class IOStream(object):
     def close(self):
         """Close this stream."""
         if self.socket is not None:
+            if any(sys.exc_info()):
+                self.error = sys.exc_info()[1]
             if self._read_until_close:
                 callback = self._read_callback
                 self._read_callback = None
@@ -239,12 +232,16 @@ class IOStream(object):
                 self._state = None
             self.socket.close()
             self.socket = None
-            if self._close_callback and self._pending_callbacks == 0:
+        self._maybe_run_close_callback()
-                # if there are pending callbacks, don't run the close callback
+
-                # until they're done (see _maybe_add_error_handler)
+    def _maybe_run_close_callback(self):
-                cb = self._close_callback
+        if (self.socket is None and self._close_callback and
-                self._close_callback = None
+            self._pending_callbacks == 0):
-                self._run_callback(cb)
+            # if there are pending callbacks, don't run the close callback
+            # until they're done (see _maybe_add_error_handler)
+            cb = self._close_callback
+            self._close_callback = None
+            self._run_callback(cb)
 
     def reading(self):
         """Returns true if we are currently reading from the stream."""
@@ -274,6 +271,9 @@ class IOStream(object):
             if not self.socket:
                 return
             if events & self.io_loop.ERROR:
+                errno = self.socket.getsockopt(socket.SOL_SOCKET,
+                                               socket.SO_ERROR)
+                self.error = socket.error(errno, os.strerror(errno))
                 # We may have queued up a user callback in _handle_read or
                 # _handle_write, so don't close the IOStream until those
                 # callbacks have had a chance to run.
@@ -332,22 +332,65 @@ class IOStream(object):
             self.io_loop.add_callback(wrapper)
 
     def _handle_read(self):
-        while True:
+        try:
             try:
-                # Read from the socket until we get EWOULDBLOCK or equivalent.
+                # Pretend to have a pending callback so that an EOF in
-                # SSL sockets do some internal buffering, and if the data is
+                # _read_to_buffer doesn't trigger an immediate close
-                # sitting in the SSL object's buffer select() and friends
+                # callback.  At the end of this method we'll either
-                # can't see it; the only way to find out if it's there is to
+                # estabilsh a real pending callback via
-                # try to read it.
+                # _read_from_buffer or run the close callback.
-                result = self._read_to_buffer()
+                #
-            except Exception:
+                # We need two try statements here so that
-                self.close()
+                # pending_callbacks is decremented before the `except`
-                return
+                # clause below (which calls `close` and does need to
-            if result == 0:
+                # trigger the callback)
-                break
+                self._pending_callbacks += 1
-            else:
+                while True:
-                if self._read_from_buffer():
+                    # Read from the socket until we get EWOULDBLOCK or equivalent.
-                    return
+                    # SSL sockets do some internal buffering, and if the data is
+                    # sitting in the SSL object's buffer select() and friends
+                    # can't see it; the only way to find out if it's there is to
+                    # try to read it.
+                    if self._read_to_buffer() == 0:
+                        break
+            finally:
+                self._pending_callbacks -= 1
+        except Exception:
+            logging.warning("error on read", exc_info=True)
+            self.close()
+            return
+        if self._read_from_buffer():
+            return
+        else:
+            self._maybe_run_close_callback()
+
+    def _set_read_callback(self, callback):
+        assert not self._read_callback, "Already reading"
+        self._read_callback = stack_context.wrap(callback)
+
+    def _try_inline_read(self):
+        """Attempt to complete the current read operation from buffered data.
+
+        If the read can be completed without blocking, schedules the
+        read callback on the next IOLoop iteration; otherwise starts
+        listening for reads on the socket.
+        """
+        # See if we've already got the data from a previous read
+        if self._read_from_buffer():
+            return
+        self._check_closed()
+        try:
+            # See comments in _handle_read about incrementing _pending_callbacks
+            self._pending_callbacks += 1
+            while True:
+                if self._read_to_buffer() == 0:
+                    break
+                self._check_closed()
+        finally:
+            self._pending_callbacks -= 1
+        if self._read_from_buffer():
+            return
+        self._add_io_state(self.io_loop.READ)
 
     def _read_from_socket(self):
         """Attempts to read from the socket.
@@ -397,20 +440,21 @@ class IOStream(object):
 
         Returns True if the read was completed.
         """
-        if self._read_bytes is not None:
+        if self._streaming_callback is not None and self._read_buffer_size:
-            if self._streaming_callback is not None and self._read_buffer_size:
+            bytes_to_consume = self._read_buffer_size
-                bytes_to_consume = min(self._read_bytes, self._read_buffer_size)
+            if self._read_bytes is not None:
+                bytes_to_consume = min(self._read_bytes, bytes_to_consume)
                 self._read_bytes -= bytes_to_consume
-                self._run_callback(self._streaming_callback,
+            self._run_callback(self._streaming_callback,
-                                   self._consume(bytes_to_consume))
+                               self._consume(bytes_to_consume))
-            if self._read_buffer_size >= self._read_bytes:
+        if self._read_bytes is not None and self._read_buffer_size >= self._read_bytes:
-                num_bytes = self._read_bytes
+            num_bytes = self._read_bytes
-                callback = self._read_callback
+            callback = self._read_callback
-                self._read_callback = None
+            self._read_callback = None
-                self._streaming_callback = None
+            self._streaming_callback = None
-                self._read_bytes = None
+            self._read_bytes = None
-                self._run_callback(callback, self._consume(num_bytes))
+            self._run_callback(callback, self._consume(num_bytes))
-                return True
+            return True
         elif self._read_delimiter is not None:
             # Multi-byte delimiters (e.g. '\r\n') may straddle two
             # chunks in the read buffer, so we can't easily find them
@@ -420,56 +464,41 @@ class IOStream(object):
             # to be in the first few chunks.  Merge the buffer gradually
             # since large merges are relatively expensive and get undone in
             # consume().
-            loc = -1
             if self._read_buffer:
-                loc = self._read_buffer[0].find(self._read_delimiter)
+                while True:
-            while loc == -1 and len(self._read_buffer) > 1:
+                    loc = self._read_buffer[0].find(self._read_delimiter)
-                # Grow by doubling, but don't split the second chunk just
+                    if loc != -1:
-                # because the first one is small.
+                        callback = self._read_callback
-                new_len = max(len(self._read_buffer[0]) * 2,
+                        delimiter_len = len(self._read_delimiter)
-                              (len(self._read_buffer[0]) +
+                        self._read_callback = None
-                               len(self._read_buffer[1])))
+                        self._streaming_callback = None
-                _merge_prefix(self._read_buffer, new_len)
+                        self._read_delimiter = None
-                loc = self._read_buffer[0].find(self._read_delimiter)
+                        self._run_callback(callback,
-            if loc != -1:
+                                           self._consume(loc + delimiter_len))
-                callback = self._read_callback
+                        return True
-                delimiter_len = len(self._read_delimiter)
+                    if len(self._read_buffer) == 1:
-                self._read_callback = None
+                        break
-                self._streaming_callback = None
+                    _double_prefix(self._read_buffer)
-                self._read_delimiter = None
-                self._run_callback(callback,
-                                   self._consume(loc + delimiter_len))
-                return True
         elif self._read_regex is not None:
-            m = None
             if self._read_buffer:
-                m = self._read_regex.search(self._read_buffer[0])
+                while True:
-            while m is None and len(self._read_buffer) > 1:
+                    m = self._read_regex.search(self._read_buffer[0])
-                # Grow by doubling, but don't split the second chunk just
+                    if m is not None:
-                # because the first one is small.
+                        callback = self._read_callback
-                new_len = max(len(self._read_buffer[0]) * 2,
+                        self._read_callback = None
-                              (len(self._read_buffer[0]) +
+                        self._streaming_callback = None
-                               len(self._read_buffer[1])))
+                        self._read_regex = None
-                _merge_prefix(self._read_buffer, new_len)
+                        self._run_callback(callback, self._consume(m.end()))
-                m = self._read_regex.search(self._read_buffer[0])
+                        return True
-            _merge_prefix(self._read_buffer, sys.maxint)
+                    if len(self._read_buffer) == 1:
-            m = self._read_regex.search(self._read_buffer[0])
+                        break
-            if m:
+                    _double_prefix(self._read_buffer)
-                callback = self._read_callback
-                self._read_callback = None
-                self._streaming_callback = None
-                self._read_regex = None
-                self._run_callback(callback, self._consume(m.end()))
-                return True
-        elif self._read_until_close:
-            if self._streaming_callback is not None and self._read_buffer_size:
-                self._run_callback(self._streaming_callback,
-                                   self._consume(self._read_buffer_size))
         return False
 
     def _handle_connect(self):
         err = self.socket.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR)
         if err != 0:
+            self.error = socket.error(err, os.strerror(err))
             # IOLoop implementations may vary: some of them return
             # an error state before the socket becomes writable, so
             # in that case a connection failure would be handled by the
@@ -537,10 +566,7 @@ class IOStream(object):
     def _maybe_add_error_listener(self):
         if self._state is None and self._pending_callbacks == 0:
             if self.socket is None:
-                cb = self._close_callback
+                self._maybe_run_close_callback()
-                if cb is not None:
-                    self._close_callback = None
-                    self._run_callback(cb)
             else:
                 self._add_io_state(ioloop.IOLoop.READ)
 
@@ -628,7 +654,7 @@ class SSLIOStream(IOStream):
                 return self.close()
             raise
         except socket.error, err:
-            if err.args[0] == errno.ECONNABORTED:
+            if err.args[0] in (errno.ECONNABORTED, errno.ECONNRESET):
                 return self.close()
         else:
             self._ssl_accepting = False
@@ -655,7 +681,6 @@ class SSLIOStream(IOStream):
         # until we've completed the SSL handshake (so certificates are
         # available, etc).
 
-
     def _read_from_socket(self):
         if self._ssl_accepting:
             # If the handshake hasn't finished yet, there can't be anything
@@ -686,6 +711,16 @@ class SSLIOStream(IOStream):
             return None
         return chunk
 
+
+def _double_prefix(deque):
+    """Grow by doubling, but don't split the second chunk just because the
+    first one is small.
+    """
+    new_len = max(len(deque[0]) * 2,
+                  (len(deque[0]) + len(deque[1])))
+    _merge_prefix(deque, new_len)
+
+
 def _merge_prefix(deque, size):
     """Replace the first entries in a deque of strings with a single
     string of up to size bytes.
@@ -723,6 +758,7 @@ def _merge_prefix(deque, size):
     if not deque:
         deque.appendleft(b(""))
 
+
 def doctests():
     import doctest
     return doctest.DocTestSuite()

libs/tornado/locale.py

@@ -39,17 +39,22 @@ supported by gettext and related tools).  If neither method is called,
 the locale.translate method will simply return the original string.
 """
 
+from __future__ import absolute_import, division, with_statement
+
 import csv
 import datetime
 import logging
 import os
 import re
 
+from tornado import escape
+
 _default_locale = "en_US"
 _translations = {}
 _supported_locales = frozenset([_default_locale])
 _use_gettext = False
 
+
 def get(*locale_codes):
     """Returns the closest match for the given locale codes.
 
@@ -109,17 +114,28 @@ def load_translations(directory):
     global _supported_locales
     _translations = {}
     for path in os.listdir(directory):
-        if not path.endswith(".csv"): continue
+        if not path.endswith(".csv"):
+            continue
         locale, extension = path.split(".")
         if not re.match("[a-z]+(_[A-Z]+)?$", locale):
             logging.error("Unrecognized locale %r (path: %s)", locale,
                           os.path.join(directory, path))
             continue
-        f = open(os.path.join(directory, path), "r")
+        full_path = os.path.join(directory, path)
+        try:
+            # python 3: csv.reader requires a file open in text mode.
+            # Force utf8 to avoid dependence on $LANG environment variable.
+            f = open(full_path, "r", encoding="utf-8")
+        except TypeError:
+            # python 2: files return byte strings, which are decoded below.
+            # Once we drop python 2.5, this could use io.open instead
+            # on both 2 and 3.
+            f = open(full_path, "r")
         _translations[locale] = {}
         for i, row in enumerate(csv.reader(f)):
-            if not row or len(row) < 2: continue
+            if not row or len(row) < 2:
-            row = [c.decode("utf-8").strip() for c in row]
+                continue
+            row = [escape.to_unicode(c).strip() for c in row]
             english, translation = row[:2]
             if len(row) > 2:
                 plural = row[2] or "unknown"
@@ -132,7 +148,8 @@ def load_translations(directory):
             _translations[locale].setdefault(plural, {})[english] = translation
         f.close()
     _supported_locales = frozenset(_translations.keys() + [_default_locale])
-    logging.info("Supported locales: %s", sorted(_supported_locales))
+    logging.debug("Supported locales: %s", sorted(_supported_locales))
+
 
 def load_gettext_translations(directory, domain):
     """Loads translations from gettext's locale tree
@@ -158,10 +175,12 @@ def load_gettext_translations(directory, domain):
     global _use_gettext
     _translations = {}
     for lang in os.listdir(directory):
-        if lang.startswith('.'): continue  # skip .svn, etc
+        if lang.startswith('.'):
-        if os.path.isfile(os.path.join(directory, lang)): continue
+            continue  # skip .svn, etc
+        if os.path.isfile(os.path.join(directory, lang)):
+            continue
         try:
-            os.stat(os.path.join(directory, lang, "LC_MESSAGES", domain+".mo"))
+            os.stat(os.path.join(directory, lang, "LC_MESSAGES", domain + ".mo"))
             _translations[lang] = gettext.translation(domain, directory,
                                                       languages=[lang])
         except Exception, e:
@@ -169,10 +188,10 @@ def load_gettext_translations(directory, domain):
             continue
     _supported_locales = frozenset(_translations.keys() + [_default_locale])
     _use_gettext = True
-    logging.info("Supported locales: %s", sorted(_supported_locales))
+    logging.debug("Supported locales: %s", sorted(_supported_locales))
 
 
-def get_supported_locales(cls):
+def get_supported_locales():
     """Returns a list of all the supported locale codes."""
     return _supported_locales
 
@@ -187,7 +206,8 @@ class Locale(object):
     def get_closest(cls, *locale_codes):
         """Returns the closest match for the given locale code."""
         for code in locale_codes:
-            if not code: continue
+            if not code:
+                continue
             code = code.replace("-", "_")
             parts = code.split("_")
             if len(parts) > 2:
@@ -289,16 +309,16 @@ class Locale(object):
             if relative and days == 0:
                 if seconds < 50:
                     return _("1 second ago", "%(seconds)d seconds ago",
-                             seconds) % { "seconds": seconds }
+                             seconds) % {"seconds": seconds}
 
                 if seconds < 50 * 60:
                     minutes = round(seconds / 60.0)
                     return _("1 minute ago", "%(minutes)d minutes ago",
-                             minutes) % { "minutes": minutes }
+                             minutes) % {"minutes": minutes}
 
                 hours = round(seconds / (60.0 * 60))
                 return _("1 hour ago", "%(hours)d hours ago",
-                         hours) % { "hours": hours }
+                         hours) % {"hours": hours}
 
             if days == 0:
                 format = _("%(time)s")
@@ -364,8 +384,10 @@ class Locale(object):
         of size 1.
         """
         _ = self.translate
-        if len(parts) == 0: return ""
+        if len(parts) == 0:
-        if len(parts) == 1: return parts[0]
+            return ""
+        if len(parts) == 1:
+            return parts[0]
         comma = u' \u0648 ' if self.code.startswith("fa") else u", "
         return _("%(commas)s and %(last)s") % {
             "commas": comma.join(parts[:-1]),
@@ -383,6 +405,7 @@ class Locale(object):
             value = value[:-3]
         return ",".join(reversed(parts))
 
+
 class CSVLocale(Locale):
     """Locale implementation using tornado's CSV translation format."""
     def translate(self, message, plural_message=None, count=None):
@@ -397,14 +420,28 @@ class CSVLocale(Locale):
             message_dict = self.translations.get("unknown", {})
         return message_dict.get(message, message)
 
+
 class GettextLocale(Locale):
     """Locale implementation using the gettext module."""
+    def __init__(self, code, translations):
+        try:
+            # python 2
+            self.ngettext = translations.ungettext
+            self.gettext = translations.ugettext
+        except AttributeError:
+            # python 3
+            self.ngettext = translations.ngettext
+            self.gettext = translations.gettext
+        # self.gettext must exist before __init__ is called, since it
+        # calls into self.translate
+        super(GettextLocale, self).__init__(code, translations)
+
     def translate(self, message, plural_message=None, count=None):
         if plural_message is not None:
             assert count is not None
-            return self.translations.ungettext(message, plural_message, count)
+            return self.ngettext(message, plural_message, count)
         else:
-            return self.translations.ugettext(message)
+            return self.gettext(message)
 
 LOCALE_NAMES = {
     "af_ZA": {"name_en": u"Afrikaans", "name": u"Afrikaans"},

libs/tornado/netutil.py

@@ -16,6 +16,8 @@
 
 """Miscellaneous network utility code."""
 
+from __future__ import absolute_import, division, with_statement
+
 import errno
 import logging
 import os
@@ -28,10 +30,11 @@ from tornado.iostream import IOStream, SSLIOStream
 from tornado.platform.auto import set_close_exec
 
 try:
-    import ssl # Python 2.6+
+    import ssl  # Python 2.6+
 except ImportError:
     ssl = None
 
+
 class TCPServer(object):
     r"""A non-blocking, single-threaded TCP server.
 
@@ -89,6 +92,23 @@ class TCPServer(object):
         self._pending_sockets = []
         self._started = False
 
+        # Verify the SSL options. Otherwise we don't get errors until clients
+        # connect. This doesn't verify that the keys are legitimate, but
+        # the SSL module doesn't do that until there is a connected socket
+        # which seems like too much work
+        if self.ssl_options is not None:
+            # Only certfile is required: it can contain both keys
+            if 'certfile' not in self.ssl_options:
+                raise KeyError('missing key "certfile" in ssl_options')
+
+            if not os.path.exists(self.ssl_options['certfile']):
+                raise ValueError('certfile "%s" does not exist' %
+                    self.ssl_options['certfile'])
+            if ('keyfile' in self.ssl_options and
+                    not os.path.exists(self.ssl_options['keyfile'])):
+                raise ValueError('keyfile "%s" does not exist' %
+                    self.ssl_options['keyfile'])
+
     def listen(self, port, address=""):
         """Starts accepting connections on the given port.
 
@@ -238,12 +258,19 @@ def bind_sockets(port, address=None, family=socket.AF_UNSPEC, backlog=128):
     if address == "":
         address = None
     flags = socket.AI_PASSIVE
+    if hasattr(socket, "AI_ADDRCONFIG"):
+        # AI_ADDRCONFIG ensures that we only try to bind on ipv6
+        # if the system is configured for it, but the flag doesn't
+        # exist on some platforms (specifically WinXP, although
+        # newer versions of windows have it)
+        flags |= socket.AI_ADDRCONFIG
     for res in set(socket.getaddrinfo(address, port, family, socket.SOCK_STREAM,
                                   0, flags)):
         af, socktype, proto, canonname, sockaddr = res
         sock = socket.socket(af, socktype, proto)
         set_close_exec(sock.fileno())
-        sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+        if os.name != 'nt':
+            sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
         if af == socket.AF_INET6:
             # On linux, ipv6 sockets accept ipv4 too by default,
             # but this makes it impossible to bind to both
@@ -291,6 +318,7 @@ if hasattr(socket, 'AF_UNIX'):
         sock.listen(backlog)
         return sock
 
+
 def add_accept_handler(sock, callback, io_loop=None):
     """Adds an ``IOLoop`` event handler to accept new connections on ``sock``.
 
@@ -302,6 +330,7 @@ def add_accept_handler(sock, callback, io_loop=None):
     """
     if io_loop is None:
         io_loop = IOLoop.instance()
+
     def accept_handler(fd, events):
         while True:
             try:

libs/tornado/options.py

@@ -48,12 +48,16 @@ kwarg to define). We also accept multi-value options. See the documentation
 for define() below.
 """
 
+from __future__ import absolute_import, division, with_statement
+
 import datetime
 import logging
 import logging.handlers
 import re
 import sys
+import os
 import time
+import textwrap
 
 from tornado.escape import _unicode
 
@@ -64,136 +68,123 @@ except ImportError:
     curses = None
 
 
-def define(name, default=None, type=None, help=None, metavar=None,
+class Error(Exception):
-           multiple=False, group=None):
+    """Exception raised by errors in the options module."""
-    """Defines a new command line option.
+    pass
-
-    If type is given (one of str, float, int, datetime, or timedelta)
-    or can be inferred from the default, we parse the command line
-    arguments based on the given type. If multiple is True, we accept
-    comma-separated values, and the option value is always a list.
-
-    For multi-value integers, we also accept the syntax x:y, which
-    turns into range(x, y) - very useful for long integer ranges.
-
-    help and metavar are used to construct the automatically generated
-    command line help string. The help message is formatted like::
-
-       --name=METAVAR      help string
-
-    group is used to group the defined options in logical groups. By default,
-    command line options are grouped by the defined file.
-
-    Command line option names must be unique globally. They can be parsed
-    from the command line with parse_command_line() or parsed from a
-    config file with parse_config_file.
-    """
-    if name in options:
-        raise Error("Option %r already defined in %s", name,
-                    options[name].file_name)
-    frame = sys._getframe(0)
-    options_file = frame.f_code.co_filename
-    file_name = frame.f_back.f_code.co_filename
-    if file_name == options_file: file_name = ""
-    if type is None:
-        if not multiple and default is not None:
-            type = default.__class__
-        else:
-            type = str
-    if group:
-        group_name = group
-    else:
-        group_name = file_name
-    options[name] = _Option(name, file_name=file_name, default=default,
-                            type=type, help=help, metavar=metavar,
-                            multiple=multiple, group_name=group_name)
-
-
-def parse_command_line(args=None):
-    """Parses all options given on the command line.
-
-    We return all command line arguments that are not options as a list.
-    """
-    if args is None: args = sys.argv
-    remaining = []
-    for i in xrange(1, len(args)):
-        # All things after the last option are command line arguments
-        if not args[i].startswith("-"):
-            remaining = args[i:]
-            break
-        if args[i] == "--":
-            remaining = args[i+1:]
-            break
-        arg = args[i].lstrip("-")
-        name, equals, value = arg.partition("=")
-        name = name.replace('-', '_')
-        if not name in options:
-            print_help()
-            raise Error('Unrecognized command line option: %r' % name)
-        option = options[name]
-        if not equals:
-            if option.type == bool:
-                value = "true"
-            else:
-                raise Error('Option %r requires a value' % name)
-        option.parse(value)
-    if options.help:
-        print_help()
-        sys.exit(0)
-
-    # Set up log level and pretty console logging by default
-    if options.logging != 'none':
-        logging.getLogger().setLevel(getattr(logging, options.logging.upper()))
-        enable_pretty_logging()
-
-    return remaining
-
-
-def parse_config_file(path):
-    """Parses and loads the Python config file at the given path."""
-    config = {}
-    execfile(path, config, config)
-    for name in config:
-        if name in options:
-            options[name].set(config[name])
-
-
-def print_help(file=sys.stdout):
-    """Prints all the command line options to stdout."""
-    print >> file, "Usage: %s [OPTIONS]" % sys.argv[0]
-    print >> file, ""
-    print >> file, "Options:"
-    by_group = {}
-    for option in options.itervalues():
-        by_group.setdefault(option.group_name, []).append(option)
-
-    for filename, o in sorted(by_group.items()):
-        if filename: print >> file, filename
-        o.sort(key=lambda option: option.name)
-        for option in o:
-            prefix = option.name
-            if option.metavar:
-                prefix += "=" + option.metavar
-            print >> file, "  --%-30s %s" % (prefix, option.help or "")
-    print >> file
 
 
 class _Options(dict):
-    """Our global program options, an dictionary with object-like access."""
+    """A collection of options, a dictionary with object-like access.
-    @classmethod
-    def instance(cls):
-        if not hasattr(cls, "_instance"):
-            cls._instance = cls()
-        return cls._instance
 
+    Normally accessed via static functions in the `tornado.options` module,
+    which reference a global instance.
+    """
     def __getattr__(self, name):
         if isinstance(self.get(name), _Option):
             return self[name].value()
         raise AttributeError("Unrecognized option %r" % name)
 
+    def __setattr__(self, name, value):
+        if isinstance(self.get(name), _Option):
+            return self[name].set(value)
+        raise AttributeError("Unrecognized option %r" % name)
+
+    def define(self, name, default=None, type=None, help=None, metavar=None,
+               multiple=False, group=None):
+        if name in self:
+            raise Error("Option %r already defined in %s", name,
+                        self[name].file_name)
+        frame = sys._getframe(0)
+        options_file = frame.f_code.co_filename
+        file_name = frame.f_back.f_code.co_filename
+        if file_name == options_file:
+            file_name = ""
+        if type is None:
+            if not multiple and default is not None:
+                type = default.__class__
+            else:
+                type = str
+        if group:
+            group_name = group
+        else:
+            group_name = file_name
+        self[name] = _Option(name, file_name=file_name, default=default,
+                             type=type, help=help, metavar=metavar,
+                             multiple=multiple, group_name=group_name)
+
+    def parse_command_line(self, args=None):
+        if args is None:
+            args = sys.argv
+        remaining = []
+        for i in xrange(1, len(args)):
+            # All things after the last option are command line arguments
+            if not args[i].startswith("-"):
+                remaining = args[i:]
+                break
+            if args[i] == "--":
+                remaining = args[i + 1:]
+                break
+            arg = args[i].lstrip("-")
+            name, equals, value = arg.partition("=")
+            name = name.replace('-', '_')
+            if not name in self:
+                print_help()
+                raise Error('Unrecognized command line option: %r' % name)
+            option = self[name]
+            if not equals:
+                if option.type == bool:
+                    value = "true"
+                else:
+                    raise Error('Option %r requires a value' % name)
+            option.parse(value)
+        if self.help:
+            print_help()
+            sys.exit(0)
+
+        # Set up log level and pretty console logging by default
+        if self.logging != 'none':
+            logging.getLogger().setLevel(getattr(logging, self.logging.upper()))
+            enable_pretty_logging()
+
+        return remaining
+
+    def parse_config_file(self, path):
+        config = {}
+        execfile(path, config, config)
+        for name in config:
+            if name in self:
+                self[name].set(config[name])
+
+    def print_help(self, file=sys.stdout):
+        """Prints all the command line options to stdout."""
+        print >> file, "Usage: %s [OPTIONS]" % sys.argv[0]
+        print >> file, "\nOptions:\n"
+        by_group = {}
+        for option in self.itervalues():
+            by_group.setdefault(option.group_name, []).append(option)
+
+        for filename, o in sorted(by_group.items()):
+            if filename:
+                print >> file, "\n%s options:\n" % os.path.normpath(filename)
+            o.sort(key=lambda option: option.name)
+            for option in o:
+                prefix = option.name
+                if option.metavar:
+                    prefix += "=" + option.metavar
+                description = option.help or ""
+                if option.default is not None and option.default != '':
+                    description += " (default %s)" % option.default
+                lines = textwrap.wrap(description, 79 - 35)
+                if len(prefix) > 30 or len(lines) == 0:
+                    lines.insert(0, '')
+                print >> file, "  --%-30s %s" % (prefix, lines[0])
+                for line in lines[1:]:
+                    print >> file, "%-34s %s" % (' ', line)
+        print >> file
+
 
 class _Option(object):
-    def __init__(self, name, default=None, type=str, help=None, metavar=None,
+    def __init__(self, name, default=None, type=basestring, help=None, metavar=None,
                  multiple=False, file_name=None, group_name=None):
         if default is None and multiple:
             default = []
@@ -215,18 +206,17 @@ class _Option(object):
             datetime.datetime: self._parse_datetime,
             datetime.timedelta: self._parse_timedelta,
             bool: self._parse_bool,
-            str: self._parse_string,
+            basestring: self._parse_string,
         }.get(self.type, self.type)
         if self.multiple:
-            if self._value is None:
+            self._value = []
-                self._value = []
             for part in value.split(","):
                 if self.type in (int, long):
                     # allow ranges of the form X:Y (inclusive at both ends)
                     lo, _, hi = part.partition(":")
                     lo = _parse(lo)
                     hi = _parse(hi) if hi else lo
-                    self._value.extend(range(lo, hi+1))
+                    self._value.extend(range(lo, hi + 1))
                 else:
                     self._value.append(_parse(part))
         else:
@@ -244,8 +234,8 @@ class _Option(object):
                                 (self.name, self.type.__name__))
         else:
             if value != None and not isinstance(value, self.type):
-                raise Error("Option %r is required to be a %s" %
+                raise Error("Option %r is required to be a %s (%s given)" %
-                            (self.name, self.type.__name__))
+                            (self.name, self.type.__name__, type(value)))
         self._value = value
 
     # Supported date/time formats in our options
@@ -313,12 +303,62 @@ class _Option(object):
         return _unicode(value)
 
 
-class Error(Exception):
+options = _Options()
-    """Exception raised by errors in the options module."""
+"""Global options dictionary.
-    pass
+
+Supports both attribute-style and dict-style access.
+"""
 
 
-def enable_pretty_logging():
+def define(name, default=None, type=None, help=None, metavar=None,
+           multiple=False, group=None):
+    """Defines a new command line option.
+
+    If type is given (one of str, float, int, datetime, or timedelta)
+    or can be inferred from the default, we parse the command line
+    arguments based on the given type. If multiple is True, we accept
+    comma-separated values, and the option value is always a list.
+
+    For multi-value integers, we also accept the syntax x:y, which
+    turns into range(x, y) - very useful for long integer ranges.
+
+    help and metavar are used to construct the automatically generated
+    command line help string. The help message is formatted like::
+
+       --name=METAVAR      help string
+
+    group is used to group the defined options in logical groups. By default,
+    command line options are grouped by the defined file.
+
+    Command line option names must be unique globally. They can be parsed
+    from the command line with parse_command_line() or parsed from a
+    config file with parse_config_file.
+    """
+    return options.define(name, default=default, type=type, help=help,
+                          metavar=metavar, multiple=multiple, group=group)
+
+
+def parse_command_line(args=None):
+    """Parses all options given on the command line (defaults to sys.argv).
+
+    Note that args[0] is ignored since it is the program name in sys.argv.
+
+    We return a list of all arguments that are not parsed as options.
+    """
+    return options.parse_command_line(args)
+
+
+def parse_config_file(path):
+    """Parses and loads the Python config file at the given path."""
+    return options.parse_config_file(path)
+
+
+def print_help(file=sys.stdout):
+    """Prints all the command line options to stdout."""
+    return options.print_help(file)
+
+
+def enable_pretty_logging(options=options):
     """Turns on formatted logging output as configured.
 
     This is called automatically by `parse_command_line`.
@@ -348,7 +388,6 @@ def enable_pretty_logging():
         root_logger.addHandler(channel)
 
 
-
 class _LogFormatter(logging.Formatter):
     def __init__(self, color, *args, **kwargs):
         logging.Formatter.__init__(self, *args, **kwargs)
@@ -366,13 +405,13 @@ class _LogFormatter(logging.Formatter):
             if (3, 0) < sys.version_info < (3, 2, 3):
                 fg_color = unicode(fg_color, "ascii")
             self._colors = {
-                logging.DEBUG: unicode(curses.tparm(fg_color, 4), # Blue
+                logging.DEBUG: unicode(curses.tparm(fg_color, 4),  # Blue
                                        "ascii"),
-                logging.INFO: unicode(curses.tparm(fg_color, 2), # Green
+                logging.INFO: unicode(curses.tparm(fg_color, 2),  # Green
                                       "ascii"),
-                logging.WARNING: unicode(curses.tparm(fg_color, 3), # Yellow
+                logging.WARNING: unicode(curses.tparm(fg_color, 3),  # Yellow
                                          "ascii"),
-                logging.ERROR: unicode(curses.tparm(fg_color, 1), # Red
+                logging.ERROR: unicode(curses.tparm(fg_color, 1),  # Red
                                        "ascii"),
             }
             self._normal = unicode(curses.tigetstr("sgr0"), "ascii")
@@ -382,6 +421,7 @@ class _LogFormatter(logging.Formatter):
             record.message = record.getMessage()
         except Exception, e:
             record.message = "Bad message (%r): %r" % (e, record.__dict__)
+        assert isinstance(record.message, basestring)  # guaranteed by logging
         record.asctime = time.strftime(
             "%y%m%d %H:%M:%S", self.converter(record.created))
         prefix = '[%(levelname)1.1s %(asctime)s %(module)s:%(lineno)d]' % \
@@ -389,7 +429,29 @@ class _LogFormatter(logging.Formatter):
         if self._color:
             prefix = (self._colors.get(record.levelno, self._normal) +
                       prefix + self._normal)
-        formatted = prefix + " " + record.message
+
+        # Encoding notes:  The logging module prefers to work with character
+        # strings, but only enforces that log messages are instances of
+        # basestring.  In python 2, non-ascii bytestrings will make
+        # their way through the logging framework until they blow up with
+        # an unhelpful decoding error (with this formatter it happens
+        # when we attach the prefix, but there are other opportunities for
+        # exceptions further along in the framework).
+        #
+        # If a byte string makes it this far, convert it to unicode to
+        # ensure it will make it out to the logs.  Use repr() as a fallback
+        # to ensure that all byte strings can be converted successfully,
+        # but don't do it by default so we don't add extra quotes to ascii
+        # bytestrings.  This is a bit of a hacky place to do this, but
+        # it's worth it since the encoding errors that would otherwise
+        # result are so useless (and tornado is fond of using utf8-encoded
+        # byte strings whereever possible).
+        try:
+            message = _unicode(record.message)
+        except UnicodeDecodeError:
+            message = repr(record.message)
+
+        formatted = prefix + " " + message
         if record.exc_info:
             if not record.exc_text:
                 record.exc_text = self.formatException(record.exc_info)
@@ -398,9 +460,6 @@ class _LogFormatter(logging.Formatter):
         return formatted.replace("\n", "\n    ")
 
 
-options = _Options.instance()
-
-
 # Default options
 define("help", type=bool, help="show this help information")
 define("logging", default="info",

libs/tornado/platform/auto.py

@@ -23,9 +23,12 @@ Most code that needs access to this functionality should do e.g.::
     from tornado.platform.auto import set_close_exec
 """
 
+from __future__ import absolute_import, division, with_statement
+
 import os
 
 if os.name == 'nt':
-    from tornado.platform.windows import set_close_exec, Waker
+    from tornado.platform.common import Waker
+    from tornado.platform.windows import set_close_exec
 else:
     from tornado.platform.posix import set_close_exec, Waker

libs/tornado/platform/common.py

@@ -0,0 +1,89 @@
+"""Lowest-common-denominator implementations of platform functionality."""
+from __future__ import absolute_import, division, with_statement
+
+import errno
+import socket
+
+from tornado.platform import interface
+from tornado.util import b
+
+
+class Waker(interface.Waker):
+    """Create an OS independent asynchronous pipe.
+
+    For use on platforms that don't have os.pipe() (or where pipes cannot
+    be passed to select()), but do have sockets.  This includes Windows
+    and Jython.
+    """
+    def __init__(self):
+        # Based on Zope async.py: http://svn.zope.org/zc.ngi/trunk/src/zc/ngi/async.py
+
+        self.writer = socket.socket()
+        # Disable buffering -- pulling the trigger sends 1 byte,
+        # and we want that sent immediately, to wake up ASAP.
+        self.writer.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
+
+        count = 0
+        while 1:
+            count += 1
+            # Bind to a local port; for efficiency, let the OS pick
+            # a free port for us.
+            # Unfortunately, stress tests showed that we may not
+            # be able to connect to that port ("Address already in
+            # use") despite that the OS picked it.  This appears
+            # to be a race bug in the Windows socket implementation.
+            # So we loop until a connect() succeeds (almost always
+            # on the first try).  See the long thread at
+            # http://mail.zope.org/pipermail/zope/2005-July/160433.html
+            # for hideous details.
+            a = socket.socket()
+            a.bind(("127.0.0.1", 0))
+            a.listen(1)
+            connect_address = a.getsockname()  # assigned (host, port) pair
+            try:
+                self.writer.connect(connect_address)
+                break    # success
+            except socket.error, detail:
+                if (not hasattr(errno, 'WSAEADDRINUSE') or
+                    detail[0] != errno.WSAEADDRINUSE):
+                    # "Address already in use" is the only error
+                    # I've seen on two WinXP Pro SP2 boxes, under
+                    # Pythons 2.3.5 and 2.4.1.
+                    raise
+                # (10048, 'Address already in use')
+                # assert count <= 2 # never triggered in Tim's tests
+                if count >= 10:  # I've never seen it go above 2
+                    a.close()
+                    self.writer.close()
+                    raise socket.error("Cannot bind trigger!")
+                # Close `a` and try again.  Note:  I originally put a short
+                # sleep() here, but it didn't appear to help or hurt.
+                a.close()
+
+        self.reader, addr = a.accept()
+        self.reader.setblocking(0)
+        self.writer.setblocking(0)
+        a.close()
+        self.reader_fd = self.reader.fileno()
+
+    def fileno(self):
+        return self.reader.fileno()
+
+    def wake(self):
+        try:
+            self.writer.send(b("x"))
+        except (IOError, socket.error):
+            pass
+
+    def consume(self):
+        try:
+            while True:
+                result = self.reader.recv(1024)
+                if not result:
+                    break
+        except (IOError, socket.error):
+            pass
+
+    def close(self):
+        self.reader.close()
+        self.writer.close()

libs/tornado/platform/interface.py

@@ -21,10 +21,14 @@ for other tornado.platform modules.  Most code should import the appropriate
 implementation from `tornado.platform.auto`.
 """
 
+from __future__ import absolute_import, division, with_statement
+
+
 def set_close_exec(fd):
     """Sets the close-on-exec bit (``FD_CLOEXEC``)for a file descriptor."""
     raise NotImplementedError()
 
+
 class Waker(object):
     """A socket-like object that can wake another thread from ``select()``.
 
@@ -53,5 +57,3 @@ class Waker(object):
     def close(self):
         """Closes the waker's file descriptor(s)."""
         raise NotImplementedError()
-
-    

libs/tornado/platform/posix.py

@@ -16,20 +16,25 @@
 
 """Posix implementations of platform-specific functionality."""
 
+from __future__ import absolute_import, division, with_statement
+
 import fcntl
 import os
 
 from tornado.platform import interface
 from tornado.util import b
 
+
 def set_close_exec(fd):
     flags = fcntl.fcntl(fd, fcntl.F_GETFD)
     fcntl.fcntl(fd, fcntl.F_SETFD, flags | fcntl.FD_CLOEXEC)
 
+
 def _set_nonblocking(fd):
     flags = fcntl.fcntl(fd, fcntl.F_GETFL)
     fcntl.fcntl(fd, fcntl.F_SETFL, flags | os.O_NONBLOCK)
 
+
 class Waker(interface.Waker):
     def __init__(self):
         r, w = os.pipe()
@@ -53,7 +58,8 @@ class Waker(interface.Waker):
         try:
             while True:
                 result = self.reader.read()
-                if not result: break;
+                if not result:
+                    break
         except IOError:
             pass
 

libs/tornado/platform/twisted.py

@@ -41,10 +41,10 @@ recommended to call::
 
 before closing the `IOLoop`.
 
-This module has been tested with Twisted versions 11.0.0 and 11.1.0.
+This module has been tested with Twisted versions 11.0.0, 11.1.0, and 12.0.0
 """
 
-from __future__ import with_statement, absolute_import
+from __future__ import absolute_import, division, with_statement
 
 import functools
 import logging
@@ -56,7 +56,7 @@ from twisted.internet.interfaces import \
 from twisted.python import failure, log
 from twisted.internet import error
 
-from zope.interface import implements
+from zope.interface import implementer
 
 import tornado
 import tornado.ioloop
@@ -66,8 +66,6 @@ from tornado.ioloop import IOLoop
 
 class TornadoDelayedCall(object):
     """DelayedCall object for Tornado."""
-    implements(IDelayedCall)
-
     def __init__(self, reactor, seconds, f, *args, **kw):
         self._reactor = reactor
         self._func = functools.partial(f, *args, **kw)
@@ -106,6 +104,9 @@ class TornadoDelayedCall(object):
 
     def active(self):
         return self._active
+# Fake class decorator for python 2.5 compatibility
+TornadoDelayedCall = implementer(IDelayedCall)(TornadoDelayedCall)
+
 
 class TornadoReactor(PosixReactorBase):
     """Twisted reactor built on the Tornado IOLoop.
@@ -117,15 +118,13 @@ class TornadoReactor(PosixReactorBase):
     timed call functionality on top of `IOLoop.add_timeout` rather than
     using the implementation in `PosixReactorBase`.
     """
-    implements(IReactorTime, IReactorFDSet)
-
     def __init__(self, io_loop=None):
         if not io_loop:
             io_loop = tornado.ioloop.IOLoop.instance()
         self._io_loop = io_loop
         self._readers = {}  # map of reader objects to fd
         self._writers = {}  # map of writer objects to fd
-        self._fds = {} # a map of fd to a (reader, writer) tuple
+        self._fds = {}  # a map of fd to a (reader, writer) tuple
         self._delayedCalls = {}
         PosixReactorBase.__init__(self)
 
@@ -294,6 +293,8 @@ class TornadoReactor(PosixReactorBase):
         self._io_loop.start()
         if self._stopped:
             self.fireSystemEvent("shutdown")
+TornadoReactor = implementer(IReactorTime, IReactorFDSet)(TornadoReactor)
+
 
 class _TestReactor(TornadoReactor):
     """Subclass of TornadoReactor for use in unittests.
@@ -319,7 +320,6 @@ class _TestReactor(TornadoReactor):
             port, protocol, interface=interface, maxPacketSize=maxPacketSize)
 
 
-
 def install(io_loop=None):
     """Install this package as the default Twisted reactor."""
     if not io_loop:

libs/tornado/platform/windows.py

@@ -1,13 +1,10 @@
 # NOTE: win32 support is currently experimental, and not recommended
 # for production use.
 
+
+from __future__ import absolute_import, division, with_statement
 import ctypes
 import ctypes.wintypes
-import socket
-import errno
-
-from tornado.platform import interface
-from tornado.util import b
 
 # See: http://msdn.microsoft.com/en-us/library/ms724935(VS.85).aspx
 SetHandleInformation = ctypes.windll.kernel32.SetHandleInformation
@@ -21,77 +18,3 @@ def set_close_exec(fd):
     success = SetHandleInformation(fd, HANDLE_FLAG_INHERIT, 0)
     if not success:
         raise ctypes.GetLastError()
-
-
-class Waker(interface.Waker):
-    """Create an OS independent asynchronous pipe"""
-    def __init__(self):
-        # Based on Zope async.py: http://svn.zope.org/zc.ngi/trunk/src/zc/ngi/async.py
-
-        self.writer = socket.socket()
-        # Disable buffering -- pulling the trigger sends 1 byte,
-        # and we want that sent immediately, to wake up ASAP.
-        self.writer.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
-
-        count = 0
-        while 1:
-            count += 1
-            # Bind to a local port; for efficiency, let the OS pick
-            # a free port for us.
-            # Unfortunately, stress tests showed that we may not
-            # be able to connect to that port ("Address already in
-            # use") despite that the OS picked it.  This appears
-            # to be a race bug in the Windows socket implementation.
-            # So we loop until a connect() succeeds (almost always
-            # on the first try).  See the long thread at
-            # http://mail.zope.org/pipermail/zope/2005-July/160433.html
-            # for hideous details.
-            a = socket.socket()
-            a.bind(("127.0.0.1", 0))
-            connect_address = a.getsockname()  # assigned (host, port) pair
-            a.listen(1)
-            try:
-                self.writer.connect(connect_address)
-                break    # success
-            except socket.error, detail:
-                if detail[0] != errno.WSAEADDRINUSE:
-                    # "Address already in use" is the only error
-                    # I've seen on two WinXP Pro SP2 boxes, under
-                    # Pythons 2.3.5 and 2.4.1.
-                    raise
-                # (10048, 'Address already in use')
-                # assert count <= 2 # never triggered in Tim's tests
-                if count >= 10:  # I've never seen it go above 2
-                    a.close()
-                    self.writer.close()
-                    raise socket.error("Cannot bind trigger!")
-                # Close `a` and try again.  Note:  I originally put a short
-                # sleep() here, but it didn't appear to help or hurt.
-                a.close()
-
-        self.reader, addr = a.accept()
-        self.reader.setblocking(0)
-        self.writer.setblocking(0)
-        a.close()
-        self.reader_fd = self.reader.fileno()
-
-    def fileno(self):
-        return self.reader.fileno()
-
-    def wake(self):
-        try:
-            self.writer.send(b("x"))
-        except IOError:
-            pass
-
-    def consume(self):
-        try:
-            while True:
-                result = self.reader.recv(1024)
-                if not result: break
-        except IOError:
-            pass
-
-    def close(self):
-        self.reader.close()
-        self.writer.close()

libs/tornado/process.py

@@ -16,6 +16,8 @@
 
 """Utilities for working with multiple processes."""
 
+from __future__ import absolute_import, division, with_statement
+
 import errno
 import logging
 import os
@@ -27,10 +29,11 @@ from binascii import hexlify
 from tornado import ioloop
 
 try:
-    import multiprocessing # Python 2.6+
+    import multiprocessing  # Python 2.6+
 except ImportError:
     multiprocessing = None
 
+
 def cpu_count():
     """Returns the number of processors on this machine."""
     if multiprocessing is not None:
@@ -45,6 +48,7 @@ def cpu_count():
     logging.error("Could not detect number of processors; assuming 1")
     return 1
 
+
 def _reseed_random():
     if 'random' not in sys.modules:
         return
@@ -61,6 +65,7 @@ def _reseed_random():
 
 _task_id = None
 
+
 def fork_processes(num_processes, max_restarts=100):
     """Starts multiple worker processes.
 
@@ -95,6 +100,7 @@ def fork_processes(num_processes, max_restarts=100):
                            "IOLoop.instance() before calling start_processes()")
     logging.info("Starting %d processes", num_processes)
     children = {}
+
     def start_child(i):
         pid = os.fork()
         if pid == 0:
@@ -108,7 +114,8 @@ def fork_processes(num_processes, max_restarts=100):
             return None
     for i in range(num_processes):
         id = start_child(i)
-        if id is not None: return id
+        if id is not None:
+            return id
     num_restarts = 0
     while children:
         try:
@@ -133,13 +140,15 @@ def fork_processes(num_processes, max_restarts=100):
         if num_restarts > max_restarts:
             raise RuntimeError("Too many child restarts, giving up")
         new_id = start_child(id)
-        if new_id is not None: return new_id
+        if new_id is not None:
+            return new_id
     # All child processes exited cleanly, so exit the master process
     # instead of just returning to right after the call to
     # fork_processes (which will probably just start up another IOLoop
     # unless the caller checks the return value).
     sys.exit(0)
 
+
 def task_id():
     """Returns the current task id, if any.
 

libs/tornado/simple_httpclient.py

@@ -1,12 +1,12 @@
 #!/usr/bin/env python
-from __future__ import with_statement
+from __future__ import absolute_import, division, with_statement
 
 from tornado.escape import utf8, _unicode, native_str
 from tornado.httpclient import HTTPRequest, HTTPResponse, HTTPError, AsyncHTTPClient, main
 from tornado.httputil import HTTPHeaders
 from tornado.iostream import IOStream, SSLIOStream
 from tornado import stack_context
-from tornado.util import b
+from tornado.util import b, GzipDecompressor
 
 import base64
 import collections
@@ -20,7 +20,6 @@ import socket
 import sys
 import time
 import urlparse
-import zlib
 
 try:
     from io import BytesIO  # python 3
@@ -28,12 +27,13 @@ except ImportError:
     from cStringIO import StringIO as BytesIO  # python 2
 
 try:
-    import ssl # python 2.6+
+    import ssl  # python 2.6+
 except ImportError:
     ssl = None
 
 _DEFAULT_CA_CERTS = os.path.dirname(__file__) + '/ca-certificates.crt'
 
+
 class SimpleAsyncHTTPClient(AsyncHTTPClient):
     """Non-blocking HTTP client with no external dependencies.
 
@@ -93,8 +93,10 @@ class SimpleAsyncHTTPClient(AsyncHTTPClient):
     def fetch(self, request, callback, **kwargs):
         if not isinstance(request, HTTPRequest):
             request = HTTPRequest(url=request, **kwargs)
-        if not isinstance(request.headers, HTTPHeaders):
+        # We're going to modify this (to add Host, Accept-Encoding, etc),
-            request.headers = HTTPHeaders(request.headers)
+        # so make sure we don't modify the caller's object.  This is also
+        # where normal dicts get converted to HTTPHeaders objects.
+        request.headers = HTTPHeaders(request.headers)
         callback = stack_context.wrap(callback)
         self.queue.append((request, callback))
         self._process_queue()
@@ -119,9 +121,8 @@ class SimpleAsyncHTTPClient(AsyncHTTPClient):
         self._process_queue()
 
 
-
 class _HTTPConnection(object):
-    _SUPPORTED_METHODS = set(["GET", "HEAD", "POST", "PUT", "DELETE"])
+    _SUPPORTED_METHODS = set(["GET", "HEAD", "POST", "PUT", "DELETE", "PATCH", "OPTIONS"])
 
     def __init__(self, io_loop, client, request, release_callback,
                  final_callback, max_buffer_size):
@@ -160,6 +161,7 @@ class _HTTPConnection(object):
             if re.match(r'^\[.*\]$', host):
                 # raw ipv6 addresses in urls are enclosed in brackets
                 host = host[1:-1]
+            parsed_hostname = host  # save final parsed host for _on_connect
             if self.client.hostname_mapping is not None:
                 host = self.client.hostname_mapping.get(host, host)
 
@@ -198,7 +200,7 @@ class _HTTPConnection(object):
                 # compatibility with servers configured for TLSv1 only,
                 # but nearly all servers support SSLv3:
                 # http://blog.ivanristic.com/2011/09/ssl-survey-protocol-support.html
-                if sys.version_info >= (2,7):
+                if sys.version_info >= (2, 7):
                     ssl_options["ciphers"] = "DEFAULT:!SSLv2"
                 else:
                     # This is really only necessary for pre-1.0 versions
@@ -218,30 +220,33 @@ class _HTTPConnection(object):
             if timeout:
                 self._timeout = self.io_loop.add_timeout(
                     self.start_time + timeout,
-                    self._on_timeout)
+                    stack_context.wrap(self._on_timeout))
             self.stream.set_close_callback(self._on_close)
             self.stream.connect(sockaddr,
-                                functools.partial(self._on_connect, parsed))
+                                functools.partial(self._on_connect, parsed,
+                                                  parsed_hostname))
 
     def _on_timeout(self):
         self._timeout = None
-        self._run_callback(HTTPResponse(self.request, 599,
+        if self.final_callback is not None:
-                                        request_time=time.time() - self.start_time,
+            raise HTTPError(599, "Timeout")
-                                        error=HTTPError(599, "Timeout")))
-        self.stream.close()
 
-    def _on_connect(self, parsed):
+    def _on_connect(self, parsed, parsed_hostname):
         if self._timeout is not None:
             self.io_loop.remove_timeout(self._timeout)
             self._timeout = None
         if self.request.request_timeout:
             self._timeout = self.io_loop.add_timeout(
                 self.start_time + self.request.request_timeout,
-                self._on_timeout)
+                stack_context.wrap(self._on_timeout))
         if (self.request.validate_cert and
             isinstance(self.stream, SSLIOStream)):
             match_hostname(self.stream.socket.getpeercert(),
-                           parsed.hostname)
+                           # ipv6 addresses are broken (in
+                           # parsed.hostname) until 2.7, here is
+                           # correctly parsed value calculated in
+                           # __init__
+                           parsed_hostname)
         if (self.request.method not in self._SUPPORTED_METHODS and
             not self.request.allow_nonstandard_methods):
             raise KeyError("unknown method %s" % self.request.method)
@@ -250,8 +255,13 @@ class _HTTPConnection(object):
                     'proxy_username', 'proxy_password'):
             if getattr(self.request, key, None):
                 raise NotImplementedError('%s not supported' % key)
+        if "Connection" not in self.request.headers:
+            self.request.headers["Connection"] = "close"
         if "Host" not in self.request.headers:
-            self.request.headers["Host"] = parsed.netloc
+            if '@' in parsed.netloc:
+                self.request.headers["Host"] = parsed.netloc.rpartition('@')[-1]
+            else:
+                self.request.headers["Host"] = parsed.netloc
         username, password = None, None
         if parsed.username is not None:
             username, password = parsed.username, parsed.password
@@ -265,7 +275,7 @@ class _HTTPConnection(object):
         if self.request.user_agent:
             self.request.headers["User-Agent"] = self.request.user_agent
         if not self.request.allow_nonstandard_methods:
-            if self.request.method in ("POST", "PUT"):
+            if self.request.method in ("POST", "PATCH", "PUT"):
                 assert self.request.body is not None
             else:
                 assert self.request.body is None
@@ -313,12 +323,12 @@ class _HTTPConnection(object):
             self._run_callback(HTTPResponse(self.request, 599, error=e,
                                 request_time=time.time() - self.start_time,
                                 ))
+            if hasattr(self, "stream"):
+                self.stream.close()
 
     def _on_close(self):
-        self._run_callback(HTTPResponse(
+        if self.final_callback is not None:
-                self.request, 599,
+            raise HTTPError(599, "Connection closed")
-                request_time=time.time() - self.start_time,
-                error=HTTPError(599, "Connection closed")))
 
     def _on_headers(self, data):
         data = native_str(data.decode("latin1"))
@@ -354,16 +364,16 @@ class _HTTPConnection(object):
         if 100 <= self.code < 200 or self.code in (204, 304):
             # These response codes never have bodies
             # http://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.3
-            assert "Transfer-Encoding" not in self.headers
+            if ("Transfer-Encoding" in self.headers or
-            assert content_length in (None, 0)
+                content_length not in (None, 0)):
+                raise ValueError("Response with code %d should not have body" %
+                                 self.code)
             self._on_body(b(""))
             return
 
         if (self.request.use_gzip and
             self.headers.get("Content-Encoding") == "gzip"):
-            # Magic parameter makes zlib module understand gzip header
+            self._decompressor = GzipDecompressor()
-            # http://stackoverflow.com/questions/1838699/how-can-i-decompress-a-gzip-stream-with-zlib
-            self._decompressor = zlib.decompressobj(16+zlib.MAX_WBITS)
         if self.headers.get("Transfer-Encoding") == "chunked":
             self.chunks = []
             self.stream.read_until(b("\r\n"), self._on_chunk_length)
@@ -405,7 +415,8 @@ class _HTTPConnection(object):
             self.stream.close()
             return
         if self._decompressor:
-            data = self._decompressor.decompress(data)
+            data = (self._decompressor.decompress(data) +
+                    self._decompressor.flush())
         if self.request.streaming_callback:
             if self.chunks is None:
                 # if chunks is not None, we already called streaming_callback
@@ -413,7 +424,7 @@ class _HTTPConnection(object):
                 self.request.streaming_callback(data)
             buffer = BytesIO()
         else:
-            buffer = BytesIO(data) # TODO: don't require one big string?
+            buffer = BytesIO(data)  # TODO: don't require one big string?
         response = HTTPResponse(original_request,
                                 self.code, headers=self.headers,
                                 request_time=time.time() - self.start_time,
@@ -426,9 +437,21 @@ class _HTTPConnection(object):
         # TODO: "chunk extensions" http://tools.ietf.org/html/rfc2616#section-3.6.1
         length = int(data.strip(), 16)
         if length == 0:
-            # all the data has been decompressed, so we don't need to
+            if self._decompressor is not None:
-            # decompress again in _on_body
+                tail = self._decompressor.flush()
-            self._decompressor = None
+                if tail:
+                    # I believe the tail will always be empty (i.e.
+                    # decompress will return all it can).  The purpose
+                    # of the flush call is to detect errors such
+                    # as truncated input.  But in case it ever returns
+                    # anything, treat it as an extra chunk
+                    if self.request.streaming_callback is not None:
+                        self.request.streaming_callback(tail)
+                    else:
+                        self.chunks.append(tail)
+                # all the data has been decompressed, so we don't need to
+                # decompress again in _on_body
+                self._decompressor = None
             self._on_body(b('').join(self.chunks))
         else:
             self.stream.read_bytes(length + 2,  # chunk ends with \r\n
@@ -452,6 +475,7 @@ class _HTTPConnection(object):
 class CertificateError(ValueError):
     pass
 
+
 def _dnsname_to_pat(dn):
     pats = []
     for frag in dn.split(r'.'):
@@ -465,6 +489,7 @@ def _dnsname_to_pat(dn):
             pats.append(frag.replace(r'\*', '[^.]*'))
     return re.compile(r'\A' + r'\.'.join(pats) + r'\Z', re.IGNORECASE)
 
+
 def match_hostname(cert, hostname):
     """Verify that *cert* (in decoded format as returned by
     SSLSocket.getpeercert()) matches the *hostname*.  RFC 2818 rules

libs/tornado/stack_context.py

@@ -66,19 +66,24 @@ Here are a few rules of thumb for when it's necessary:
   block that references your `StackContext`.
 '''
 
-from __future__ import with_statement
+from __future__ import absolute_import, division, with_statement
 
 import contextlib
 import functools
 import itertools
+import operator
 import sys
 import threading
 
+from tornado.util import raise_exc_info
+
+
 class _State(threading.local):
     def __init__(self):
         self.contexts = ()
 _state = _State()
 
+
 class StackContext(object):
     '''Establishes the given context as a StackContext that will be transferred.
 
@@ -91,24 +96,33 @@ class StackContext(object):
     StackContext takes the function itself rather than its result::
 
       with StackContext(my_context):
+
+    The result of ``with StackContext() as cb:`` is a deactivation
+    callback.  Run this callback when the StackContext is no longer
+    needed to ensure that it is not propagated any further (note that
+    deactivating a context does not affect any instances of that
+    context that are currently pending).  This is an advanced feature
+    and not necessary in most applications.
     '''
-    def __init__(self, context_factory):
+    def __init__(self, context_factory, _active_cell=None):
         self.context_factory = context_factory
+        self.active_cell = _active_cell or [True]
 
     # Note that some of this code is duplicated in ExceptionStackContext
     # below.  ExceptionStackContext is more common and doesn't need
     # the full generality of this class.
     def __enter__(self):
         self.old_contexts = _state.contexts
-        # _state.contexts is a tuple of (class, arg) pairs
+        # _state.contexts is a tuple of (class, arg, active_cell) tuples
         _state.contexts = (self.old_contexts +
-                           ((StackContext, self.context_factory),))
+                           ((StackContext, self.context_factory, self.active_cell),))
         try:
             self.context = self.context_factory()
             self.context.__enter__()
         except Exception:
             _state.contexts = self.old_contexts
             raise
+        return lambda: operator.setitem(self.active_cell, 0, False)
 
     def __exit__(self, type, value, traceback):
         try:
@@ -116,6 +130,7 @@ class StackContext(object):
         finally:
             _state.contexts = self.old_contexts
 
+
 class ExceptionStackContext(object):
     '''Specialization of StackContext for exception handling.
 
@@ -129,13 +144,16 @@ class ExceptionStackContext(object):
     If the exception handler returns true, the exception will be
     consumed and will not be propagated to other exception handlers.
     '''
-    def __init__(self, exception_handler):
+    def __init__(self, exception_handler, _active_cell=None):
         self.exception_handler = exception_handler
+        self.active_cell = _active_cell or [True]
 
     def __enter__(self):
         self.old_contexts = _state.contexts
         _state.contexts = (self.old_contexts +
-                           ((ExceptionStackContext, self.exception_handler),))
+                           ((ExceptionStackContext, self.exception_handler,
+                             self.active_cell),))
+        return lambda: operator.setitem(self.active_cell, 0, False)
 
     def __exit__(self, type, value, traceback):
         try:
@@ -144,6 +162,7 @@ class ExceptionStackContext(object):
         finally:
             _state.contexts = self.old_contexts
 
+
 class NullContext(object):
     '''Resets the StackContext.
 
@@ -158,9 +177,11 @@ class NullContext(object):
     def __exit__(self, type, value, traceback):
         _state.contexts = self.old_contexts
 
+
 class _StackContextWrapper(functools.partial):
     pass
 
+
 def wrap(fn):
     '''Returns a callable object that will restore the current StackContext
     when executed.
@@ -173,12 +194,17 @@ def wrap(fn):
         return fn
     # functools.wraps doesn't appear to work on functools.partial objects
     #@functools.wraps(fn)
-    def wrapped(callback, contexts, *args, **kwargs):
+
+    def wrapped(*args, **kwargs):
+        callback, contexts, args = args[0], args[1], args[2:]
+        
         if contexts is _state.contexts or not contexts:
             callback(*args, **kwargs)
             return
         if not _state.contexts:
-            new_contexts = [cls(arg) for (cls, arg) in contexts]
+            new_contexts = [cls(arg, active_cell)
+                            for (cls, arg, active_cell) in contexts
+                            if active_cell[0]]
         # If we're moving down the stack, _state.contexts is a prefix
         # of contexts.  For each element of contexts not in that prefix,
         # create a new StackContext object.
@@ -190,10 +216,13 @@ def wrap(fn):
                 for a, b in itertools.izip(_state.contexts, contexts))):
             # contexts have been removed or changed, so start over
             new_contexts = ([NullContext()] +
-                            [cls(arg) for (cls,arg) in contexts])
+                            [cls(arg, active_cell)
+                             for (cls, arg, active_cell) in contexts
+                             if active_cell[0]])
         else:
-            new_contexts = [cls(arg)
+            new_contexts = [cls(arg, active_cell)
-                            for (cls, arg) in contexts[len(_state.contexts):]]
+                            for (cls, arg, active_cell) in contexts[len(_state.contexts):]
+                            if active_cell[0]]
         if len(new_contexts) > 1:
             with _nested(*new_contexts):
                 callback(*args, **kwargs)
@@ -207,6 +236,7 @@ def wrap(fn):
     else:
         return _StackContextWrapper(fn)
 
+
 @contextlib.contextmanager
 def _nested(*managers):
     """Support multiple context managers in a single with-statement.
@@ -240,5 +270,4 @@ def _nested(*managers):
             # Don't rely on sys.exc_info() still containing
             # the right information. Another exception may
             # have been raised and caught by an exit method
-            raise exc[0], exc[1], exc[2]
+            raise_exc_info(exc)
-

libs/tornado/template.py

@@ -101,6 +101,11 @@ with ``{# ... #}``.
 
         {% apply linkify %}{{name}} said: {{message}}{% end %}
 
+    Note that as an implementation detail apply blocks are implemented
+    as nested functions and thus may interact strangely with variables
+    set via ``{% set %}``, or the use of ``{% break %}`` or ``{% continue %}``
+    within loops.
+
 ``{% autoescape *function* %}``
     Sets the autoescape mode for the current file.  This does not affect
     other files, even those referenced by ``{% include %}``.  Note that
@@ -134,7 +139,8 @@ with ``{# ... #}``.
     tag will be ignored.  For an example, see the ``{% block %}`` tag.
 
 ``{% for *var* in *expr* %}...{% end %}``
-    Same as the python ``for`` statement.
+    Same as the python ``for`` statement.  ``{% break %}`` and
+    ``{% continue %}`` may be used inside the loop.
 
 ``{% from *x* import *y* %}``
     Same as the python ``import`` statement.
@@ -165,14 +171,15 @@ with ``{# ... #}``.
 ``{% set *x* = *y* %}``
     Sets a local variable.
 
-``{% try %}...{% except %}...{% finally %}...{% end %}``
+``{% try %}...{% except %}...{% finally %}...{% else %}...{% end %}``
     Same as the python ``try`` statement.
 
 ``{% while *condition* %}... {% end %}``
-    Same as the python ``while`` statement.
+    Same as the python ``while`` statement.  ``{% break %}`` and
+    ``{% continue %}`` may be used inside the loop.
 """
 
-from __future__ import with_statement
+from __future__ import absolute_import, division, with_statement
 
 import cStringIO
 import datetime
@@ -189,6 +196,7 @@ from tornado.util import bytes_type, ObjectDict
 _DEFAULT_AUTOESCAPE = "xhtml_escape"
 _UNSET = object()
 
+
 class Template(object):
     """A compiled template.
 
@@ -217,7 +225,7 @@ class Template(object):
             # the module name used in __name__ below.
             self.compiled = compile(
                 escape.to_unicode(self.code),
-                "%s.generated.py" % self.name.replace('.','_'),
+                "%s.generated.py" % self.name.replace('.', '_'),
                 "exec")
         except Exception:
             formatted_code = _format_code(self.code).rstrip()
@@ -326,6 +334,7 @@ class BaseLoader(object):
     def _create_template(self, name):
         raise NotImplementedError()
 
+
 class Loader(BaseLoader):
     """A template loader that loads from a single root directory.
 
@@ -350,7 +359,7 @@ class Loader(BaseLoader):
 
     def _create_template(self, name):
         path = os.path.join(self.root, name)
-        f = open(path, "r")
+        f = open(path, "rb")
         template = Template(f.read(), name=name, loader=self)
         f.close()
         return template
@@ -404,7 +413,6 @@ class _File(_Node):
         return (self.body,)
 
 
-
 class _ChunkList(_Node):
     def __init__(self, chunks):
         self.chunks = chunks
@@ -531,11 +539,13 @@ class _Expression(_Node):
                               writer.current_template.autoescape, self.line)
         writer.write_line("_append(_tmp)", self.line)
 
+
 class _Module(_Expression):
     def __init__(self, expression, line):
         super(_Module, self).__init__("_modules." + expression, line,
                                       raw=True)
 
+
 class _Text(_Node):
     def __init__(self, value, line):
         self.value = value
@@ -608,7 +618,7 @@ class _CodeWriter(object):
             ancestors = ["%s:%d" % (tmpl.name, lineno)
                          for (tmpl, lineno) in self.include_stack]
             line_comment += ' (via %s)' % ', '.join(reversed(ancestors))
-        print >> self.file, "    "*indent + line + line_comment
+        print >> self.file, "    " * indent + line + line_comment
 
 
 class _TemplateReader(object):
@@ -651,9 +661,12 @@ class _TemplateReader(object):
         if type(key) is slice:
             size = len(self)
             start, stop, step = key.indices(size)
-            if start is None: start = self.pos
+            if start is None:
-            else: start += self.pos
+                start = self.pos
-            if stop is not None: stop += self.pos
+            else:
+                start += self.pos
+            if stop is not None:
+                stop += self.pos
             return self.text[slice(start, stop, step)]
         elif key < 0:
             return self.text[key]
@@ -670,7 +683,7 @@ def _format_code(code):
     return "".join([format % (i + 1, line) for (i, line) in enumerate(lines)])
 
 
-def _parse(reader, template, in_block=None):
+def _parse(reader, template, in_block=None, in_loop=None):
     body = _ChunkList([])
     while True:
         # Find next template directive
@@ -751,7 +764,7 @@ def _parse(reader, template, in_block=None):
 
         # Intermediate ("else", "elif", etc) blocks
         intermediate_blocks = {
-            "else": set(["if", "for", "while"]),
+            "else": set(["if", "for", "while", "try"]),
             "elif": set(["if"]),
             "except": set(["try"]),
             "finally": set(["try"]),
@@ -796,7 +809,8 @@ def _parse(reader, template, in_block=None):
                 block = _Statement(suffix, line)
             elif operator == "autoescape":
                 fn = suffix.strip()
-                if fn == "None": fn = None
+                if fn == "None":
+                    fn = None
                 template.autoescape = fn
                 continue
             elif operator == "raw":
@@ -808,7 +822,15 @@ def _parse(reader, template, in_block=None):
 
         elif operator in ("apply", "block", "try", "if", "for", "while"):
             # parse inner body recursively
-            block_body = _parse(reader, template, operator)
+            if operator in ("for", "while"):
+                block_body = _parse(reader, template, operator, operator)
+            elif operator == "apply":
+                # apply creates a nested function so syntactically it's not
+                # in the loop.
+                block_body = _parse(reader, template, operator, None)
+            else:
+                block_body = _parse(reader, template, operator, in_loop)
+
             if operator == "apply":
                 if not suffix:
                     raise ParseError("apply missing method name on line %d" % line)
@@ -822,5 +844,11 @@ def _parse(reader, template, in_block=None):
             body.chunks.append(block)
             continue
 
+        elif operator in ("break", "continue"):
+            if not in_loop:
+                raise ParseError("%s outside %s block" % (operator, set(["for", "while"])))
+            body.chunks.append(_Statement(contents, line))
+            continue
+
         else:
             raise ParseError("unknown operator: %r" % operator)

libs/tornado/testing.py

@@ -18,12 +18,13 @@ inheritance.  See the docstrings for each class/function below for more
 information.
 """
 
-from __future__ import with_statement
+from __future__ import absolute_import, division, with_statement
 
 from cStringIO import StringIO
 try:
     from tornado.httpclient import AsyncHTTPClient
     from tornado.httpserver import HTTPServer
+    from tornado.simple_httpclient import SimpleAsyncHTTPClient
     from tornado.ioloop import IOLoop
 except ImportError:
     # These modules are not importable on app engine.  Parts of this module
@@ -31,15 +32,20 @@ except ImportError:
     AsyncHTTPClient = None
     HTTPServer = None
     IOLoop = None
+    SimpleAsyncHTTPClient = None
 from tornado.stack_context import StackContext, NullContext
+from tornado.util import raise_exc_info
 import contextlib
 import logging
+import os
 import signal
 import sys
 import time
 import unittest
 
 _next_port = 10000
+
+
 def get_unused_port():
     """Returns a (hopefully) unused port number."""
     global _next_port
@@ -47,6 +53,7 @@ def get_unused_port():
     _next_port = _next_port + 1
     return port
 
+
 class AsyncTestCase(unittest.TestCase):
     """TestCase subclass for testing IOLoop-based asynchronous code.
 
@@ -104,6 +111,7 @@ class AsyncTestCase(unittest.TestCase):
         self.__running = False
         self.__failure = None
         self.__stop_args = None
+        self.__timeout = None
 
     def setUp(self):
         super(AsyncTestCase, self).setUp()
@@ -134,9 +142,18 @@ class AsyncTestCase(unittest.TestCase):
             self.__failure = sys.exc_info()
             self.stop()
 
+    def __rethrow(self):
+        if self.__failure is not None:
+            failure = self.__failure
+            self.__failure = None
+            raise_exc_info(failure)
+
     def run(self, result=None):
         with StackContext(self._stack_context):
             super(AsyncTestCase, self).run(result)
+        # In case an exception escaped super.run or the StackContext caught
+        # an exception when there wasn't a wait() to re-raise it, do so here.
+        self.__rethrow()
 
     def stop(self, _arg=None, **kwargs):
         '''Stops the ioloop, causing one pending (or future) call to wait()
@@ -165,12 +182,14 @@ class AsyncTestCase(unittest.TestCase):
                 def timeout_func():
                     try:
                         raise self.failureException(
-                          'Async operation timed out after %d seconds' %
+                          'Async operation timed out after %s seconds' %
                           timeout)
                     except Exception:
                         self.__failure = sys.exc_info()
                     self.stop()
-                self.io_loop.add_timeout(time.time() + timeout, timeout_func)
+                if self.__timeout is not None:
+                    self.io_loop.remove_timeout(self.__timeout)
+                self.__timeout = self.io_loop.add_timeout(time.time() + timeout, timeout_func)
             while True:
                 self.__running = True
                 with NullContext():
@@ -183,13 +202,7 @@ class AsyncTestCase(unittest.TestCase):
                     break
         assert self.__stopped
         self.__stopped = False
-        if self.__failure is not None:
+        self.__rethrow()
-            # 2to3 isn't smart enough to convert three-argument raise
-            # statements correctly in some cases.
-            if isinstance(self.__failure[1], self.__failure[0]):
-                raise self.__failure[1], None, self.__failure[2]
-            else:
-                raise self.__failure[0], self.__failure[1], self.__failure[2]
         result = self.__stop_args
         self.__stop_args = None
         return result
@@ -222,12 +235,19 @@ class AsyncHTTPTestCase(AsyncTestCase):
         super(AsyncHTTPTestCase, self).setUp()
         self.__port = None
 
-        self.http_client = AsyncHTTPClient(io_loop=self.io_loop)
+        self.http_client = self.get_http_client()
         self._app = self.get_app()
-        self.http_server = HTTPServer(self._app, io_loop=self.io_loop,
+        self.http_server = self.get_http_server()
-                                      **self.get_httpserver_options())
         self.http_server.listen(self.get_http_port(), address="127.0.0.1")
 
+    def get_http_client(self):
+        return AsyncHTTPClient(io_loop=self.io_loop)
+
+    def get_http_server(self):
+        return HTTPServer(self._app, io_loop=self.io_loop,
+                          **self.get_httpserver_options())
+
+
     def get_app(self):
         """Should be overridden by subclasses to return a
         tornado.web.Application or other HTTPServer callback.
@@ -247,12 +267,12 @@ class AsyncHTTPTestCase(AsyncTestCase):
 
     def get_httpserver_options(self):
         """May be overridden by subclasses to return additional
-        keyword arguments for HTTPServer.
+        keyword arguments for the server.
         """
         return {}
 
     def get_http_port(self):
-        """Returns the port used by the HTTPServer.
+        """Returns the port used by the server.
 
         A new port is chosen for each test.
         """
@@ -260,15 +280,53 @@ class AsyncHTTPTestCase(AsyncTestCase):
             self.__port = get_unused_port()
         return self.__port
 
+    def get_protocol(self):
+        return 'http'
+
     def get_url(self, path):
         """Returns an absolute url for the given path on the test server."""
-        return 'http://localhost:%s%s' % (self.get_http_port(), path)
+        return '%s://localhost:%s%s' % (self.get_protocol(),
+                                        self.get_http_port(), path)
 
     def tearDown(self):
         self.http_server.stop()
         self.http_client.close()
         super(AsyncHTTPTestCase, self).tearDown()
 
+
+class AsyncHTTPSTestCase(AsyncHTTPTestCase):
+    """A test case that starts an HTTPS server.
+
+    Interface is generally the same as `AsyncHTTPTestCase`.
+    """
+    def get_http_client(self):
+        # Some versions of libcurl have deadlock bugs with ssl,
+        # so always run these tests with SimpleAsyncHTTPClient.
+        return SimpleAsyncHTTPClient(io_loop=self.io_loop, force_instance=True)
+
+    def get_httpserver_options(self):
+        return dict(ssl_options=self.get_ssl_options())
+
+    def get_ssl_options(self):
+        """May be overridden by subclasses to select SSL options.
+
+        By default includes a self-signed testing certificate.
+        """
+        # Testing keys were generated with:
+        # openssl req -new -keyout tornado/test/test.key -out tornado/test/test.crt -nodes -days 3650 -x509
+        module_dir = os.path.dirname(__file__)
+        return dict(
+                certfile=os.path.join(module_dir, 'test', 'test.crt'),
+                keyfile=os.path.join(module_dir, 'test', 'test.key'))
+
+    def get_protocol(self):
+        return 'https'
+
+    def fetch(self, path, **kwargs):
+        return AsyncHTTPTestCase.fetch(self, path, validate_cert=False,
+                   **kwargs)
+
+
 class LogTrapTestCase(unittest.TestCase):
     """A test case that captures and discards all logging output
     if the test passes.
@@ -308,7 +366,8 @@ class LogTrapTestCase(unittest.TestCase):
         finally:
             handler.stream = old_stream
 
-def main():
+
+def main(**kwargs):
     """A simple test runner.
 
     This test runner is essentially equivalent to `unittest.main` from
@@ -329,10 +388,15 @@ def main():
     be overridden by naming a single test on the command line::
 
         # Runs all tests
-        tornado/test/runtests.py
+        python -m tornado.test.runtests
         # Runs one test
-        tornado/test/runtests.py tornado.test.stack_context_test
+        python -m tornado.test.runtests tornado.test.stack_context_test
 
+    Additional keyword arguments passed through to ``unittest.main()``.
+    For example, use ``tornado.testing.main(verbosity=2)``
+    to show many test details as they are run.
+    See http://docs.python.org/library/unittest.html#unittest.main
+    for full argument list.
     """
     from tornado.options import define, options, parse_command_line
 
@@ -364,9 +428,9 @@ def main():
         # test discovery, which is incompatible with auto2to3), so don't
         # set module if we're not asking for a specific test.
         if len(argv) > 1:
-            unittest.main(module=None, argv=argv)
+            unittest.main(module=None, argv=argv, **kwargs)
         else:
-            unittest.main(defaultTest="all", argv=argv)
+            unittest.main(defaultTest="all", argv=argv, **kwargs)
     except SystemExit, e:
         if e.code == 0:
             logging.info('PASS')

libs/tornado/util.py

@@ -1,5 +1,10 @@
 """Miscellaneous utility functions."""
 
+from __future__ import absolute_import, division, with_statement
+
+import zlib
+
+
 class ObjectDict(dict):
     """Makes a dictionary behave like an object."""
     def __getattr__(self, name):
@@ -12,6 +17,36 @@ class ObjectDict(dict):
         self[name] = value
 
 
+class GzipDecompressor(object):
+    """Streaming gzip decompressor.
+
+    The interface is like that of `zlib.decompressobj` (without the
+    optional arguments, but it understands gzip headers and checksums.
+    """
+    def __init__(self):
+        # Magic parameter makes zlib module understand gzip header
+        # http://stackoverflow.com/questions/1838699/how-can-i-decompress-a-gzip-stream-with-zlib
+        # This works on cpython and pypy, but not jython.
+        self.decompressobj = zlib.decompressobj(16 + zlib.MAX_WBITS)
+
+    def decompress(self, value):
+        """Decompress a chunk, returning newly-available data.
+
+        Some data may be buffered for later processing; `flush` must
+        be called when there is no more input data to ensure that
+        all data was processed.
+        """
+        return self.decompressobj.decompress(value)
+
+    def flush(self):
+        """Return any remaining buffered data not yet returned by decompress.
+
+        Also checks for errors such as truncated input.
+        No other methods may be called on this object after `flush`.
+        """
+        return self.decompressobj.flush()
+
+
 def import_object(name):
     """Imports an object by name.
 
@@ -42,6 +77,25 @@ else:
         return s
     bytes_type = str
 
+
+def raise_exc_info(exc_info):
+    """Re-raise an exception (with original traceback) from an exc_info tuple.
+
+    The argument is a ``(type, value, traceback)`` tuple as returned by
+    `sys.exc_info`.
+    """
+    # 2to3 isn't smart enough to convert three-argument raise
+    # statements correctly in some cases.
+    if isinstance(exc_info[1], exc_info[0]):
+        raise exc_info[1], None, exc_info[2]
+        # After 2to3: raise exc_info[1].with_traceback(exc_info[2])
+    else:
+        # I think this branch is only taken for string exceptions,
+        # which were removed in Python 2.6.
+        raise exc_info[0], exc_info[1], exc_info[2]
+        # After 2to3: raise exc_info[0](exc_info[1]).with_traceback(exc_info[2])
+
+
 def doctests():
     import doctest
     return doctest.DocTestSuite()

libs/tornado/web.py

@@ -49,7 +49,7 @@ threads it is important to use IOLoop.add_callback to transfer control
 back to the main thread before finishing the request.
 """
 
-from __future__ import with_statement
+from __future__ import absolute_import, division, with_statement
 
 import Cookie
 import base64
@@ -83,13 +83,14 @@ from tornado import locale
 from tornado import stack_context
 from tornado import template
 from tornado.escape import utf8, _unicode
-from tornado.util import b, bytes_type, import_object, ObjectDict
+from tornado.util import b, bytes_type, import_object, ObjectDict, raise_exc_info
 
 try:
     from io import BytesIO  # python 3
 except ImportError:
     from cStringIO import StringIO as BytesIO  # python 2
 
+
 class RequestHandler(object):
     """Subclass this class and define get() or post() to make a handler.
 
@@ -97,7 +98,8 @@ class RequestHandler(object):
     should override the class variable SUPPORTED_METHODS in your
     RequestHandler class.
     """
-    SUPPORTED_METHODS = ("GET", "HEAD", "POST", "DELETE", "PUT", "OPTIONS")
+    SUPPORTED_METHODS = ("GET", "HEAD", "POST", "DELETE", "PATCH", "PUT",
+                         "OPTIONS")
 
     _template_loaders = {}  # {path: template.BaseLoader}
     _template_loader_lock = threading.Lock()
@@ -121,7 +123,7 @@ class RequestHandler(object):
         self.ui["modules"] = self.ui["_modules"]
         self.clear()
         # Check since connection is not available in WSGI
-        if hasattr(self.request, "connection"):
+        if getattr(self.request, "connection", None):
             self.request.connection.stream.set_close_callback(
                 self.on_connection_close)
         self.initialize(**kwargs)
@@ -164,6 +166,9 @@ class RequestHandler(object):
     def delete(self, *args, **kwargs):
         raise HTTPError(405)
 
+    def patch(self, *args, **kwargs):
+        raise HTTPError(405)
+
     def put(self, *args, **kwargs):
         raise HTTPError(405)
 
@@ -259,6 +264,15 @@ class RequestHandler(object):
         """
         self._list_headers.append((name, self._convert_header_value(value)))
 
+    def clear_header(self, name):
+        """Clears an outgoing header, undoing a previous `set_header` call.
+
+        Note that this method does not apply to multi-valued headers
+        set by `add_header`.
+        """
+        if name in self._headers:
+            del self._headers[name]
+
     def _convert_header_value(self, value):
         if isinstance(value, bytes_type):
             pass
@@ -279,8 +293,8 @@ class RequestHandler(object):
             raise ValueError("Unsafe header value %r", value)
         return value
 
-
     _ARG_DEFAULT = []
+
     def get_argument(self, name, default=_ARG_DEFAULT, strip=True):
         """Returns the value of the argument with the given name.
 
@@ -358,25 +372,27 @@ class RequestHandler(object):
         if re.search(r"[\x00-\x20]", name + value):
             # Don't let us accidentally inject bad stuff
             raise ValueError("Invalid cookie %r: %r" % (name, value))
-        if not hasattr(self, "_new_cookies"):
+        if not hasattr(self, "_new_cookie"):
-            self._new_cookies = []
+            self._new_cookie = Cookie.SimpleCookie()
-        new_cookie = Cookie.SimpleCookie()
+        if name in self._new_cookie:
-        self._new_cookies.append(new_cookie)
+            del self._new_cookie[name]
-        new_cookie[name] = value
+        self._new_cookie[name] = value
+        morsel = self._new_cookie[name]
         if domain:
-            new_cookie[name]["domain"] = domain
+            morsel["domain"] = domain
         if expires_days is not None and not expires:
             expires = datetime.datetime.utcnow() + datetime.timedelta(
                 days=expires_days)
         if expires:
             timestamp = calendar.timegm(expires.utctimetuple())
-            new_cookie[name]["expires"] = email.utils.formatdate(
+            morsel["expires"] = email.utils.formatdate(
                 timestamp, localtime=False, usegmt=True)
         if path:
-            new_cookie[name]["path"] = path
+            morsel["path"] = path
         for k, v in kwargs.iteritems():
-            if k == 'max_age': k = 'max-age'
+            if k == 'max_age':
-            new_cookie[name][k] = v
+                k = 'max-age'
+            morsel[k] = v
 
     def clear_cookie(self, name, path="/", domain=None):
         """Deletes the cookie with the given name."""
@@ -401,6 +417,9 @@ class RequestHandler(object):
         Note that the ``expires_days`` parameter sets the lifetime of the
         cookie in the browser, but is independent of the ``max_age_days``
         parameter to `get_secure_cookie`.
+
+        Secure cookies may contain arbitrary byte values, not just unicode
+        strings (unlike regular cookies)
         """
         self.set_cookie(name, self.create_signed_value(name, value),
                         expires_days=expires_days, **kwargs)
@@ -417,9 +436,14 @@ class RequestHandler(object):
                                    name, value)
 
     def get_secure_cookie(self, name, value=None, max_age_days=31):
-        """Returns the given signed cookie if it validates, or None."""
+        """Returns the given signed cookie if it validates, or None.
+
+        The decoded cookie value is returned as a byte string (unlike
+        `get_cookie`).
+        """
         self.require_setting("cookie_secret", "secure cookies")
-        if value is None: value = self.get_cookie(name)
+        if value is None:
+            value = self.get_cookie(name)
         return decode_signed_value(self.application.settings["cookie_secret"],
                                    name, value, max_age_days=max_age_days)
 
@@ -482,7 +506,8 @@ class RequestHandler(object):
         html_bodies = []
         for module in getattr(self, "_active_modules", {}).itervalues():
             embed_part = module.embedded_javascript()
-            if embed_part: js_embed.append(utf8(embed_part))
+            if embed_part:
+                js_embed.append(utf8(embed_part))
             file_part = module.javascript_files()
             if file_part:
                 if isinstance(file_part, (unicode, bytes_type)):
@@ -490,7 +515,8 @@ class RequestHandler(object):
                 else:
                     js_files.extend(file_part)
             embed_part = module.embedded_css()
-            if embed_part: css_embed.append(utf8(embed_part))
+            if embed_part:
+                css_embed.append(utf8(embed_part))
             file_part = module.css_files()
             if file_part:
                 if isinstance(file_part, (unicode, bytes_type)):
@@ -498,9 +524,12 @@ class RequestHandler(object):
                 else:
                     css_files.extend(file_part)
             head_part = module.html_head()
-            if head_part: html_heads.append(utf8(head_part))
+            if head_part:
+                html_heads.append(utf8(head_part))
             body_part = module.html_body()
-            if body_part: html_bodies.append(utf8(body_part))
+            if body_part:
+                html_bodies.append(utf8(body_part))
+
         def is_absolute(path):
             return any(path.startswith(x) for x in ["/", "http:", "https:"])
         if js_files:
@@ -579,7 +608,7 @@ class RequestHandler(object):
             _=self.locale.translate,
             static_url=self.static_url,
             xsrf_form_html=self.xsrf_form_html,
-            reverse_url=self.application.reverse_url
+            reverse_url=self.reverse_url
         )
         args.update(self.ui)
         args.update(kwargs)
@@ -596,7 +625,6 @@ class RequestHandler(object):
             kwargs["autoescape"] = settings["autoescape"]
         return template.Loader(template_path, **kwargs)
 
-
     def flush(self, include_footers=False, callback=None):
         """Flushes the current output buffer to the network.
 
@@ -614,8 +642,9 @@ class RequestHandler(object):
         if not self._headers_written:
             self._headers_written = True
             for transform in self._transforms:
-                self._headers, chunk = transform.transform_first_chunk(
+                self._status_code, self._headers, chunk = \
-                    self._headers, chunk, include_footers)
+                    transform.transform_first_chunk(
+                    self._status_code, self._headers, chunk, include_footers)
             headers = self._generate_headers()
         else:
             for transform in self._transforms:
@@ -624,11 +653,11 @@ class RequestHandler(object):
 
         # Ignore the chunk and only write the headers for HEAD requests
         if self.request.method == "HEAD":
-            if headers: self.request.write(headers, callback=callback)
+            if headers:
+                self.request.write(headers, callback=callback)
             return
 
-        if headers or chunk:
+        self.request.write(headers + chunk, callback=callback)
-            self.request.write(headers + chunk, callback=callback)
 
     def finish(self, chunk=None):
         """Finishes this response, ending the HTTP request."""
@@ -637,7 +666,8 @@ class RequestHandler(object):
                                "by using async operations without the "
                                "@asynchronous decorator.")
 
-        if chunk is not None: self.write(chunk)
+        if chunk is not None:
+            self.write(chunk)
 
         # Automatically support ETags and add the Content-Length header if
         # we have not flushed any content yet.
@@ -647,13 +677,15 @@ class RequestHandler(object):
                 "Etag" not in self._headers):
                 etag = self.compute_etag()
                 if etag is not None:
+                    self.set_header("Etag", etag)
                     inm = self.request.headers.get("If-None-Match")
                     if inm and inm.find(etag) != -1:
                         self._write_buffer = []
                         self.set_status(304)
-                    else:
+            if self._status_code == 304:
-                        self.set_header("Etag", etag)
+                assert not self._write_buffer, "Cannot send body with 304"
-            if "Content-Length" not in self._headers:
+                self._clear_headers_for_304()
+            elif "Content-Length" not in self._headers:
                 content_length = sum(len(part) for part in self._write_buffer)
                 self.set_header("Content-Length", content_length)
 
@@ -720,7 +752,7 @@ class RequestHandler(object):
                 kwargs['exception'] = exc_info[1]
                 try:
                     # Put the traceback into sys.exc_info()
-                    raise exc_info[0], exc_info[1], exc_info[2]
+                    raise_exc_info(exc_info)
                 except Exception:
                     self.finish(self.get_error_html(status_code, **kwargs))
             else:
@@ -926,6 +958,7 @@ class RequestHandler(object):
             return None
         if args or kwargs:
             callback = functools.partial(callback, *args, **kwargs)
+
         def wrapper(*args, **kwargs):
             try:
                 return callback(*args, **kwargs)
@@ -964,7 +997,7 @@ class RequestHandler(object):
             # the exception value instead of the full triple,
             # so re-raise the exception to ensure that it's in
             # sys.exc_info()
-            raise type, value, traceback
+            raise_exc_info((type, value, traceback))
         except Exception:
             self._handle_request_exception(value)
         return True
@@ -984,7 +1017,7 @@ class RequestHandler(object):
             if not self._finished:
                 args = [self.decode_argument(arg) for arg in args]
                 kwargs = dict((k, self.decode_argument(v, name=k))
-                              for (k,v) in kwargs.iteritems())
+                              for (k, v) in kwargs.iteritems())
                 getattr(self, self.request.method.lower())(*args, **kwargs)
                 if self._auto_finish and not self._finished:
                     self.finish()
@@ -997,8 +1030,8 @@ class RequestHandler(object):
                       " " + httplib.responses[self._status_code])]
         lines.extend([(utf8(n) + b(": ") + utf8(v)) for n, v in
                       itertools.chain(self._headers.iteritems(), self._list_headers)])
-        for cookie_dict in getattr(self, "_new_cookies", []):
+        if hasattr(self, "_new_cookie"):
-            for cookie in cookie_dict.values():
+            for cookie in self._new_cookie.values():
                 lines.append(utf8("Set-Cookie: " + cookie.OutputString(None)))
         return b("\r\n").join(lines) + b("\r\n\r\n")
 
@@ -1044,6 +1077,17 @@ class RequestHandler(object):
     def _ui_method(self, method):
         return lambda *args, **kwargs: method(self, *args, **kwargs)
 
+    def _clear_headers_for_304(self):
+        # 304 responses should not contain entity headers (defined in
+        # http://www.w3.org/Protocols/rfc2616/rfc2616-sec7.html#sec7.1)
+        # not explicitly allowed by
+        # http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3.5
+        headers = ["Allow", "Content-Encoding", "Content-Language",
+                   "Content-Length", "Content-MD5", "Content-Range",
+                   "Content-Type", "Last-Modified"]
+        for h in headers:
+            self.clear_header(h)
+
 
 def asynchronous(method):
     """Wrap request handler methods with this if they are asynchronous.
@@ -1088,8 +1132,9 @@ def removeslash(method):
             if self.request.method in ("GET", "HEAD"):
                 uri = self.request.path.rstrip("/")
                 if uri:  # don't try to redirect '/' to ''
-                    if self.request.query: uri += "?" + self.request.query
+                    if self.request.query:
-                    self.redirect(uri)
+                        uri += "?" + self.request.query
+                    self.redirect(uri, permanent=True)
                     return
             else:
                 raise HTTPError(404)
@@ -1109,8 +1154,9 @@ def addslash(method):
         if not self.request.path.endswith("/"):
             if self.request.method in ("GET", "HEAD"):
                 uri = self.request.path + "/"
-                if self.request.query: uri += "?" + self.request.query
+                if self.request.query:
-                self.redirect(uri)
+                    uri += "?" + self.request.query
+                self.redirect(uri, permanent=True)
                 return
             raise HTTPError(404)
         return method(self, *args, **kwargs)
@@ -1162,9 +1208,15 @@ class Application(object):
 
     .. attribute:: settings
 
-       Additonal keyword arguments passed to the constructor are saved in the
+       Additional keyword arguments passed to the constructor are saved in the
        `settings` dictionary, and are often referred to in documentation as
        "application settings".
+
+    .. attribute:: debug
+
+       If `True` the application runs in debug mode, described in
+       :ref:`debug-mode`. This is an application setting in the `settings`
+       dictionary, so handlers can access it.
     """
     def __init__(self, handlers=None, default_host="", transforms=None,
                  wsgi=False, **settings):
@@ -1200,7 +1252,8 @@ class Application(object):
                             r"/(favicon\.ico)", r"/(robots\.txt)"]:
                 handlers.insert(0, (pattern, static_handler_class,
                                     static_handler_args))
-        if handlers: self.add_handlers(".*$", handlers)
+        if handlers:
+            self.add_handlers(".*$", handlers)
 
         # Automatically reload modified modules
         if self.settings.get("debug") and not wsgi:
@@ -1292,7 +1345,8 @@ class Application(object):
             self._load_ui_methods(dict((n, getattr(methods, n))
                                        for n in dir(methods)))
         elif isinstance(methods, list):
-            for m in methods: self._load_ui_methods(m)
+            for m in methods:
+                self._load_ui_methods(m)
         else:
             for name, fn in methods.iteritems():
                 if not name.startswith("_") and hasattr(fn, "__call__") \
@@ -1304,7 +1358,8 @@ class Application(object):
             self._load_ui_modules(dict((n, getattr(modules, n))
                                        for n in dir(modules)))
         elif isinstance(modules, list):
-            for m in modules: self._load_ui_modules(m)
+            for m in modules:
+                self._load_ui_modules(m)
         else:
             assert isinstance(modules, dict)
             for name, cls in modules.iteritems():
@@ -1333,7 +1388,8 @@ class Application(object):
                         # None-safe wrapper around url_unescape to handle
                         # unmatched optional groups correctly
                         def unquote(s):
-                            if s is None: return s
+                            if s is None:
+                                return s
                             return escape.url_unescape(s, encoding=None)
                         # Pass matched groups to the handler.  Since
                         # match.groups() includes both named and unnamed groups,
@@ -1343,7 +1399,7 @@ class Application(object):
 
                         if spec.regex.groupindex:
                             kwargs = dict(
-                                (k, unquote(v))
+                                (str(k), unquote(v))
                                 for (k, v) in match.groupdict().iteritems())
                         else:
                             args = [unquote(s) for s in match.groups()]
@@ -1365,7 +1421,11 @@ class Application(object):
     def reverse_url(self, name, *args):
         """Returns a URL path for handler named `name`
 
-        The handler must be added to the application as a named URLSpec
+        The handler must be added to the application as a named URLSpec.
+
+        Args will be substituted for capturing groups in the URLSpec regex.
+        They will be converted to strings if necessary, encoded as utf8,
+        and url-escaped.
         """
         if name in self.named_handlers:
             return self.named_handlers[name].reverse(*args)
@@ -1393,7 +1453,6 @@ class Application(object):
                    handler._request_summary(), request_time)
 
 
-
 class HTTPError(Exception):
     """An exception that will turn into an HTTP error response."""
     def __init__(self, status_code, log_message=None, *args):
@@ -1455,7 +1514,7 @@ class StaticFileHandler(RequestHandler):
     /static/images/myimage.png?v=xxx. Override ``get_cache_time`` method for
     more fine-grained cache control.
     """
-    CACHE_MAX_AGE = 86400*365*10 #10 years
+    CACHE_MAX_AGE = 86400 * 365 * 10  # 10 years
 
     _static_hashes = {}
     _lock = threading.Lock()  # protects _static_hashes
@@ -1639,8 +1698,8 @@ class OutputTransform(object):
     def __init__(self, request):
         pass
 
-    def transform_first_chunk(self, headers, chunk, finishing):
+    def transform_first_chunk(self, status_code, headers, chunk, finishing):
-        return headers, chunk
+        return status_code, headers, chunk
 
     def transform_chunk(self, chunk, finishing):
         return chunk
@@ -1661,7 +1720,7 @@ class GZipContentEncoding(OutputTransform):
         self._gzipping = request.supports_http_1_1() and \
             "gzip" in request.headers.get("Accept-Encoding", "")
 
-    def transform_first_chunk(self, headers, chunk, finishing):
+    def transform_first_chunk(self, status_code, headers, chunk, finishing):
         if self._gzipping:
             ctype = _unicode(headers.get("Content-Type", "")).split(";")[0]
             self._gzipping = (ctype in self.CONTENT_TYPES) and \
@@ -1675,7 +1734,7 @@ class GZipContentEncoding(OutputTransform):
             chunk = self.transform_chunk(chunk, finishing)
             if "Content-Length" in headers:
                 headers["Content-Length"] = str(len(chunk))
-        return headers, chunk
+        return status_code, headers, chunk
 
     def transform_chunk(self, chunk, finishing):
         if self._gzipping:
@@ -1698,15 +1757,17 @@ class ChunkedTransferEncoding(OutputTransform):
     def __init__(self, request):
         self._chunking = request.supports_http_1_1()
 
-    def transform_first_chunk(self, headers, chunk, finishing):
+    def transform_first_chunk(self, status_code, headers, chunk, finishing):
-        if self._chunking:
+        # 304 responses have no body (not even a zero-length body), and so
+        # should not have either Content-Length or Transfer-Encoding headers.
+        if self._chunking and status_code != 304:
             # No need to chunk the output if a Content-Length is specified
             if "Content-Length" in headers or "Transfer-Encoding" in headers:
                 self._chunking = False
             else:
                 headers["Transfer-Encoding"] = "chunked"
                 chunk = self.transform_chunk(chunk, finishing)
-        return headers, chunk
+        return status_code, headers, chunk
 
     def transform_chunk(self, block, finishing):
         if self._chunking:
@@ -1786,14 +1847,17 @@ class UIModule(object):
         """Renders a template and returns it as a string."""
         return self.handler.render_string(path, **kwargs)
 
+
 class _linkify(UIModule):
     def render(self, text, **kwargs):
         return escape.linkify(text, **kwargs)
 
+
 class _xsrf_form_html(UIModule):
     def render(self):
         return self.handler.xsrf_form_html()
 
+
 class TemplateModule(UIModule):
     """UIModule that simply renders the given template.
 
@@ -1862,10 +1926,9 @@ class TemplateModule(UIModule):
         return "".join(self._get_resources("html_body"))
 
 
-
 class URLSpec(object):
     """Specifies mappings between URLs and handlers."""
-    def __init__(self, pattern, handler_class, kwargs={}, name=None):
+    def __init__(self, pattern, handler_class, kwargs=None, name=None):
         """Creates a URLSpec.
 
         Parameters:
@@ -1889,7 +1952,7 @@ class URLSpec(object):
             ("groups in url regexes must either be all named or all "
              "positional: %r" % self.regex.pattern)
         self.handler_class = handler_class
-        self.kwargs = kwargs
+        self.kwargs = kwargs or {}
         self.name = name
         self._path, self._group_count = self._find_groups()
 
@@ -1928,7 +1991,12 @@ class URLSpec(object):
             "not found"
         if not len(args):
             return self._path
-        return self._path % tuple([str(a) for a in args])
+        converted_args = []
+        for a in args:
+            if not isinstance(a, (unicode, bytes_type)):
+                a = str(a)
+            converted_args.append(escape.url_escape(utf8(a)))
+        return self._path % tuple(converted_args)
 
 url = URLSpec
 
@@ -1938,13 +2006,14 @@ def _time_independent_equals(a, b):
         return False
     result = 0
     if type(a[0]) is int:  # python3 byte strings
-        for x, y in zip(a,b):
+        for x, y in zip(a, b):
             result |= x ^ y
     else:  # python2
         for x, y in zip(a, b):
             result |= ord(x) ^ ord(y)
     return result == 0
 
+
 def create_signed_value(secret, name, value):
     timestamp = utf8(str(int(time.time())))
     value = base64.b64encode(utf8(value))
@@ -1952,10 +2021,13 @@ def create_signed_value(secret, name, value):
     value = b("|").join([value, timestamp, signature])
     return value
 
+
 def decode_signed_value(secret, name, value, max_age_days=31):
-    if not value: return None
+    if not value:
+        return None
     parts = utf8(value).split(b("|"))
-    if len(parts) != 3: return None
+    if len(parts) != 3:
+        return None
     signature = _create_signature(secret, name, parts[0], parts[1])
     if not _time_independent_equals(parts[2], signature):
         logging.warning("Invalid cookie signature %r", value)
@@ -1974,12 +2046,15 @@ def decode_signed_value(secret, name, value, max_age_days=31):
         return None
     if parts[1].startswith(b("0")):
         logging.warning("Tampered cookie %r", value)
+        return None
     try:
         return base64.b64decode(parts[0])
     except Exception:
         return None
 
+
 def _create_signature(secret, *parts):
     hash = hmac.new(utf8(secret), digestmod=hashlib.sha1)
-    for part in parts: hash.update(utf8(part))
+    for part in parts:
+        hash.update(utf8(part))
     return utf8(hash.hexdigest())

libs/tornado/websocket.py

@@ -16,6 +16,8 @@ communication between the browser and server.
    overriding `WebSocketHandler.allow_draft76` (see that method's
    documentation for caveats).
 """
+
+from __future__ import absolute_import, division, with_statement
 # Author: Jacob Kristhammar, 2010
 
 import array
@@ -30,6 +32,7 @@ import tornado.web
 
 from tornado.util import bytes_type, b
 
+
 class WebSocketHandler(tornado.web.RequestHandler):
     """Subclass this class to create a basic WebSocket handler.
 
@@ -249,6 +252,7 @@ class WebSocketProtocol(object):
         """
         if args or kwargs:
             callback = functools.partial(callback, *args, **kwargs)
+
         def wrapper(*args, **kwargs):
             try:
                 return callback(*args, **kwargs)
@@ -471,7 +475,7 @@ class WebSocketProtocol13(WebSocketProtocol):
         sha1 = hashlib.sha1()
         sha1.update(tornado.escape.utf8(
                 self.request.headers.get("Sec-Websocket-Key")))
-        sha1.update(b("258EAFA5-E914-47DA-95CA-C5AB0DC85B11")) # Magic value
+        sha1.update(b("258EAFA5-E914-47DA-95CA-C5AB0DC85B11"))  # Magic value
         return tornado.escape.native_str(base64.b64encode(sha1.digest()))
 
     def _accept_connection(self):
@@ -552,12 +556,12 @@ class WebSocketProtocol13(WebSocketProtocol):
             self.stream.read_bytes(8, self._on_frame_length_64)
 
     def _on_frame_length_16(self, data):
-        self._frame_length = struct.unpack("!H", data)[0];
+        self._frame_length = struct.unpack("!H", data)[0]
-        self.stream.read_bytes(4, self._on_masking_key);
+        self.stream.read_bytes(4, self._on_masking_key)
 
     def _on_frame_length_64(self, data):
-        self._frame_length = struct.unpack("!Q", data)[0];
+        self._frame_length = struct.unpack("!Q", data)[0]
-        self.stream.read_bytes(4, self._on_masking_key);
+        self.stream.read_bytes(4, self._on_masking_key)
 
     def _on_masking_key(self, data):
         self._frame_mask = array.array("B", data)
@@ -604,9 +608,9 @@ class WebSocketProtocol13(WebSocketProtocol):
         if not self.client_terminated:
             self._receive_frame()
 
-
     def _handle_message(self, opcode, data):
-        if self.client_terminated: return
+        if self.client_terminated:
+            return
 
         if opcode == 0x1:
             # UTF-8 data

libs/tornado/wsgi.py

@@ -29,8 +29,9 @@ provides WSGI support in two ways:
   and Tornado handlers in a single server.
 """
 
+from __future__ import absolute_import, division, with_statement
+
 import Cookie
-import cgi
 import httplib
 import logging
 import sys
@@ -41,14 +42,37 @@ import urllib
 from tornado import escape
 from tornado import httputil
 from tornado import web
-from tornado.escape import native_str, utf8
+from tornado.escape import native_str, utf8, parse_qs_bytes
-from tornado.util import b
+from tornado.util import b, bytes_type
 
 try:
     from io import BytesIO  # python 3
 except ImportError:
     from cStringIO import StringIO as BytesIO  # python 2
 
+
+# PEP 3333 specifies that WSGI on python 3 generally deals with byte strings
+# that are smuggled inside objects of type unicode (via the latin1 encoding).
+# These functions are like those in the tornado.escape module, but defined
+# here to minimize the temptation to use them in non-wsgi contexts.
+if str is unicode:
+    def to_wsgi_str(s):
+        assert isinstance(s, bytes_type)
+        return s.decode('latin1')
+
+    def from_wsgi_str(s):
+        assert isinstance(s, str)
+        return s.encode('latin1')
+else:
+    def to_wsgi_str(s):
+        assert isinstance(s, bytes_type)
+        return s
+
+    def from_wsgi_str(s):
+        assert isinstance(s, str)
+        return s
+
+
 class WSGIApplication(web.Application):
     """A WSGI equivalent of `tornado.web.Application`.
 
@@ -92,12 +116,12 @@ class WSGIApplication(web.Application):
         assert handler._finished
         status = str(handler._status_code) + " " + \
             httplib.responses[handler._status_code]
-        headers = handler._headers.items()
+        headers = handler._headers.items() + handler._list_headers
-        for cookie_dict in getattr(handler, "_new_cookies", []):
+        if hasattr(handler, "_new_cookie"):
-            for cookie in cookie_dict.values():
+            for cookie in handler._new_cookie.values():
                 headers.append(("Set-Cookie", cookie.OutputString(None)))
         start_response(status,
-                       [(native_str(k), native_str(v)) for (k,v) in headers])
+                       [(native_str(k), native_str(v)) for (k, v) in headers])
         return handler._write_buffer
 
 
@@ -106,17 +130,18 @@ class HTTPRequest(object):
     def __init__(self, environ):
         """Parses the given WSGI environ to construct the request."""
         self.method = environ["REQUEST_METHOD"]
-        self.path = urllib.quote(environ.get("SCRIPT_NAME", ""))
+        self.path = urllib.quote(from_wsgi_str(environ.get("SCRIPT_NAME", "")))
-        self.path += urllib.quote(environ.get("PATH_INFO", ""))
+        self.path += urllib.quote(from_wsgi_str(environ.get("PATH_INFO", "")))
         self.uri = self.path
         self.arguments = {}
         self.query = environ.get("QUERY_STRING", "")
         if self.query:
             self.uri += "?" + self.query
-            arguments = cgi.parse_qs(self.query)
+            arguments = parse_qs_bytes(native_str(self.query))
             for name, values in arguments.iteritems():
                 values = [v for v in values if v]
-                if values: self.arguments[name] = values
+                if values:
+                    self.arguments[name] = values
         self.version = "HTTP/1.1"
         self.headers = httputil.HTTPHeaders()
         if environ.get("CONTENT_TYPE"):
@@ -140,18 +165,8 @@ class HTTPRequest(object):
 
         # Parse request body
         self.files = {}
-        content_type = self.headers.get("Content-Type", "")
+        httputil.parse_body_arguments(self.headers.get("Content-Type", ""),
-        if content_type.startswith("application/x-www-form-urlencoded"):
+                                      self.body, self.arguments, self.files)
-            for name, values in cgi.parse_qs(self.body).iteritems():
-                self.arguments.setdefault(name, []).extend(values)
-        elif content_type.startswith("multipart/form-data"):
-            if 'boundary=' in content_type:
-                boundary = content_type.split('boundary=',1)[1]
-                if boundary:
-                    httputil.parse_multipart_form_data(
-                        utf8(boundary), self.body, self.arguments, self.files)
-            else:
-                logging.warning("Invalid multipart/form-data")
 
         self._start_time = time.time()
         self._finish_time = None
@@ -215,6 +230,7 @@ class WSGIContainer(object):
     def __call__(self, request):
         data = {}
         response = []
+
         def start_response(status, response_headers, exc_info=None):
             data["status"] = status
             data["headers"] = response_headers
@@ -225,11 +241,12 @@ class WSGIContainer(object):
         body = b("").join(response)
         if hasattr(app_response, "close"):
             app_response.close()
-        if not data: raise Exception("WSGI app did not call start_response")
+        if not data:
+            raise Exception("WSGI app did not call start_response")
 
         status_code = int(data["status"].split()[0])
         headers = data["headers"]
-        header_set = set(k.lower() for (k,v) in headers)
+        header_set = set(k.lower() for (k, v) in headers)
         body = escape.utf8(body)
         if "content-length" not in header_set:
             headers.append(("Content-Length", str(len(body))))
@@ -261,7 +278,7 @@ class WSGIContainer(object):
         environ = {
             "REQUEST_METHOD": request.method,
             "SCRIPT_NAME": "",
-            "PATH_INFO": urllib.unquote(request.path),
+            "PATH_INFO": to_wsgi_str(escape.url_unescape(request.path, encoding=None)),
             "QUERY_STRING": request.query,
             "REMOTE_ADDR": request.remote_ip,
             "SERVER_NAME": host,