Tornado 3.2.1

libs/tornado/web.py

@@ -74,9 +74,11 @@ import traceback
 import types
 import uuid
 
-from tornado.concurrent import Future
+from tornado.concurrent import Future, is_future
 from tornado import escape
+from tornado import gen
 from tornado import httputil
+from tornado import iostream
 from tornado import locale
 from tornado.log import access_log, app_log, gen_log
 from tornado import stack_context
@@ -105,6 +107,39 @@ except ImportError:
     from urllib.parse import urlencode  # py3
 
 
+MIN_SUPPORTED_SIGNED_VALUE_VERSION = 1
+"""The oldest signed value version supported by this version of Tornado.
+
+Signed values older than this version cannot be decoded.
+
+.. versionadded:: 3.2.1
+"""
+
+MAX_SUPPORTED_SIGNED_VALUE_VERSION = 2
+"""The newest signed value version supported by this version of Tornado.
+
+Signed values newer than this version cannot be decoded.
+
+.. versionadded:: 3.2.1
+"""
+
+DEFAULT_SIGNED_VALUE_VERSION = 2
+"""The signed value version produced by `.RequestHandler.create_signed_value`.
+
+May be overridden by passing a ``version`` keyword argument.
+
+.. versionadded:: 3.2.1
+"""
+
+DEFAULT_SIGNED_VALUE_MIN_VERSION = 1
+"""The oldest signed value accepted by `.RequestHandler.get_secure_cookie`.
+
+May be overrided by passing a ``min_version`` keyword argument.
+
+.. versionadded:: 3.2.1
+"""
+
+
 class RequestHandler(object):
     """Subclass this class and define `get()` or `post()` to make a handler.
 
@@ -128,6 +163,7 @@ class RequestHandler(object):
         self._finished = False
         self._auto_finish = True
         self._transforms = None  # will be set in _execute
+        self._prepared_future = None
         self.path_args = None
         self.path_kwargs = None
         self.ui = ObjectDict((n, self._ui_method(m)) for n, m in
@@ -141,10 +177,7 @@ class RequestHandler(object):
                                                     application.ui_modules)
         self.ui["modules"] = self.ui["_tt_modules"]
         self.clear()
-        # Check since connection is not available in WSGI
-        if getattr(self.request, "connection", None):
-            self.request.connection.set_close_callback(
-                self.on_connection_close)
+        self.request.connection.set_close_callback(self.on_connection_close)
         self.initialize(**kwargs)
 
     def initialize(self):
@@ -235,7 +268,9 @@ class RequestHandler(object):
         may not be called promptly after the end user closes their
         connection.
         """
-        pass
+        if _has_stream_request_body(self.__class__):
+            if not self.request.body.done():
+                self.request.body.set_exception(iostream.StreamClosedError())
 
     def clear(self):
         """Resets all headers and content for this response."""
@@ -245,12 +280,6 @@ class RequestHandler(object):
             "Date": httputil.format_timestamp(time.time()),
         })
         self.set_default_headers()
-        if (not self.request.supports_http_1_1() and
-            getattr(self.request, 'connection', None) and
-                not self.request.connection.no_keep_alive):
-            conn_header = self.request.headers.get("Connection")
-            if conn_header and (conn_header.lower() == "keep-alive"):
-                self._headers["Connection"] = "Keep-Alive"
         self._write_buffer = []
         self._status_code = 200
         self._reason = httputil.responses[200]
@@ -455,7 +484,7 @@ class RequestHandler(object):
 
     @property
     def cookies(self):
-        """An alias for `self.request.cookies <.httpserver.HTTPRequest.cookies>`."""
+        """An alias for `self.request.cookies <.httputil.HTTPServerRequest.cookies>`."""
         return self.request.cookies
 
     def get_cookie(self, name, default=None):
@@ -524,7 +553,8 @@ class RequestHandler(object):
         for name in self.request.cookies:
             self.clear_cookie(name, path=path, domain=domain)
 
-    def set_secure_cookie(self, name, value, expires_days=30, **kwargs):
+    def set_secure_cookie(self, name, value, expires_days=30, version=None,
+                          **kwargs):
         """Signs and timestamps a cookie so it cannot be forged.
 
         You must specify the ``cookie_secret`` setting in your Application
@@ -539,32 +569,50 @@ class RequestHandler(object):
 
         Secure cookies may contain arbitrary byte values, not just unicode
         strings (unlike regular cookies)
+
+        .. versionchanged:: 3.2.1
+
+           Added the ``version`` argument.  Introduced cookie version 2
+           and made it the default.
         """
-        self.set_cookie(name, self.create_signed_value(name, value),
+        self.set_cookie(name, self.create_signed_value(name, value,
+                                                       version=version),
                         expires_days=expires_days, **kwargs)
 
-    def create_signed_value(self, name, value):
+    def create_signed_value(self, name, value, version=None):
         """Signs and timestamps a string so it cannot be forged.
 
         Normally used via set_secure_cookie, but provided as a separate
         method for non-cookie uses.  To decode a value not stored
         as a cookie use the optional value argument to get_secure_cookie.
+
+        .. versionchanged:: 3.2.1
+
+           Added the ``version`` argument.  Introduced cookie version 2
+           and made it the default.
         """
         self.require_setting("cookie_secret", "secure cookies")
         return create_signed_value(self.application.settings["cookie_secret"],
-                                   name, value)
+                                   name, value, version=version)
 
-    def get_secure_cookie(self, name, value=None, max_age_days=31):
+    def get_secure_cookie(self, name, value=None, max_age_days=31,
+                          min_version=None):
         """Returns the given signed cookie if it validates, or None.
 
         The decoded cookie value is returned as a byte string (unlike
         `get_cookie`).
+
+        .. versionchanged:: 3.2.1
+
+           Added the ``min_version`` argument.  Introduced cookie version 2;
+           both versions 1 and 2 are accepted by default.
         """
         self.require_setting("cookie_secret", "secure cookies")
         if value is None:
             value = self.get_cookie(name)
         return decode_signed_value(self.application.settings["cookie_secret"],
-                                   name, value, max_age_days=max_age_days)
+                                   name, value, max_age_days=max_age_days,
+                                   min_version=min_version)
 
     def redirect(self, url, permanent=False, status=None):
         """Sends a redirect to the given (optionally relative) URL.
@@ -598,12 +646,15 @@ class RequestHandler(object):
         Note that lists are not converted to JSON because of a potential
         cross-site security vulnerability.  All JSON output should be
         wrapped in a dictionary.  More details at
-        http://haacked.com/archive/2008/11/20/anatomy-of-a-subtle-json-vulnerability.aspx
+        http://haacked.com/archive/2009/06/25/json-hijacking.aspx/ and
+        https://github.com/facebook/tornado/issues/1009
         """
         if self._finished:
             raise RuntimeError("Cannot write() after finish().  May be caused "
                                "by using async operations without the "
                                "@asynchronous decorator.")
+        if not isinstance(chunk, (bytes_type, unicode_type, dict)):
+            raise TypeError("write() only accepts bytes, unicode, and dict objects")
         if isinstance(chunk, dict):
             chunk = escape.json_encode(chunk)
             self.set_header("Content-Type", "application/json; charset=UTF-8")
@@ -769,14 +820,10 @@ class RequestHandler(object):
         Note that only one flush callback can be outstanding at a time;
         if another flush occurs before the previous flush's callback
         has been run, the previous callback will be discarded.
-        """
-        if self.application._wsgi:
-            # WSGI applications cannot usefully support flush, so just make
-            # it a no-op (and run the callback immediately).
-            if callback is not None:
-                callback()
-            return
 
+        .. versionchanged:: 3.3
+           Now returns a `.Future` if no callback is given.
+        """
         chunk = b"".join(self._write_buffer)
         self._write_buffer = []
         if not self._headers_written:
@@ -785,19 +832,32 @@ class RequestHandler(object):
                 self._status_code, self._headers, chunk = \
                     transform.transform_first_chunk(
                         self._status_code, self._headers, chunk, include_footers)
-            headers = self._generate_headers()
+            # Ignore the chunk and only write the headers for HEAD requests
+            if self.request.method == "HEAD":
+                chunk = None
+
+            # Finalize the cookie headers (which have been stored in a side
+            # object so an outgoing cookie could be overwritten before it
+            # is sent).
+            if hasattr(self, "_new_cookie"):
+                for cookie in self._new_cookie.values():
+                    self.add_header("Set-Cookie", cookie.OutputString(None))
+
+            start_line = httputil.ResponseStartLine(self.request.version,
+                                                    self._status_code,
+                                                    self._reason)
+            return self.request.connection.write_headers(
+                start_line, self._headers, chunk, callback=callback)
         else:
             for transform in self._transforms:
                 chunk = transform.transform_chunk(chunk, include_footers)
-            headers = b""
-
-        # Ignore the chunk and only write the headers for HEAD requests
-        if self.request.method == "HEAD":
-            if headers:
-                self.request.write(headers, callback=callback)
-            return
-
-        self.request.write(headers + chunk, callback=callback)
+            # Ignore the chunk and only write the headers for HEAD requests
+            if self.request.method != "HEAD":
+                return self.request.connection.write(chunk, callback=callback)
+            else:
+                future = Future()
+                future.set_result(None)
+                return future
 
     def finish(self, chunk=None):
         """Finishes this response, ending the HTTP request."""
@@ -833,10 +893,9 @@ class RequestHandler(object):
             # are keepalive connections)
             self.request.connection.set_close_callback(None)
 
-        if not self.application._wsgi:
-            self.flush(include_footers=True)
-            self.request.finish()
-            self._log()
+        self.flush(include_footers=True)
+        self.request.finish()
+        self._log()
         self._finished = True
         self.on_finish()
         # Break up a reference cycle between this handler and the
@@ -1194,6 +1253,7 @@ class RequestHandler(object):
             self._handle_request_exception(value)
         return True
 
+    @gen.coroutine
     def _execute(self, transforms, *args, **kwargs):
         """Executes this request with the given output transforms."""
         self._transforms = transforms
@@ -1208,52 +1268,52 @@ class RequestHandler(object):
             if self.request.method not in ("GET", "HEAD", "OPTIONS") and \
                     self.application.settings.get("xsrf_cookies"):
                 self.check_xsrf_cookie()
-            self._when_complete(self.prepare(), self._execute_method)
-        except Exception as e:
-            self._handle_request_exception(e)
 
-    def _when_complete(self, result, callback):
-        try:
-            if result is None:
-                callback()
-            elif isinstance(result, Future):
-                if result.done():
-                    if result.result() is not None:
-                        raise ValueError('Expected None, got %r' % result.result())
-                    callback()
-                else:
-                    # Delayed import of IOLoop because it's not available
-                    # on app engine
-                    from tornado.ioloop import IOLoop
-                    IOLoop.current().add_future(
-                        result, functools.partial(self._when_complete,
-                                                  callback=callback))
-            else:
-                raise ValueError("Expected Future or None, got %r" % result)
-        except Exception as e:
-            self._handle_request_exception(e)
+            result = self.prepare()
+            if is_future(result):
+                result = yield result
+            if result is not None:
+                raise TypeError("Expected None, got %r" % result)
+            if self._prepared_future is not None:
+                # Tell the Application we've finished with prepare()
+                # and are ready for the body to arrive.
+                self._prepared_future.set_result(None)
+            if self._finished:
+                return
+
+            if _has_stream_request_body(self.__class__):
+                # In streaming mode request.body is a Future that signals
+                # the body has been completely received.  The Future has no
+                # result; the data has been passed to self.data_received
+                # instead.
+                try:
+                    yield self.request.body
+                except iostream.StreamClosedError:
+                    return
 
-    def _execute_method(self):
-        if not self._finished:
             method = getattr(self, self.request.method.lower())
-            self._when_complete(method(*self.path_args, **self.path_kwargs),
-                                self._execute_finish)
+            result = method(*self.path_args, **self.path_kwargs)
+            if is_future(result):
+                result = yield result
+            if result is not None:
+                raise TypeError("Expected None, got %r" % result)
+            if self._auto_finish and not self._finished:
+                self.finish()
+        except Exception as e:
+            self._handle_request_exception(e)
+            if (self._prepared_future is not None and
+                    not self._prepared_future.done()):
+                # In case we failed before setting _prepared_future, do it
+                # now (to unblock the HTTP server).  Note that this is not
+                # in a finally block to avoid GC issues prior to Python 3.4.
+                self._prepared_future.set_result(None)
 
-    def _execute_finish(self):
-        if self._auto_finish and not self._finished:
-            self.finish()
+    def data_received(self, chunk):
+        """Implement this method to handle streamed request data.
 
-    def _generate_headers(self):
-        reason = self._reason
-        lines = [utf8(self.request.version + " " +
-                      str(self._status_code) +
-                      " " + reason)]
-        lines.extend([utf8(n) + b": " + utf8(v) for n, v in self._headers.get_all()])
-
-        if hasattr(self, "_new_cookie"):
-            for cookie in self._new_cookie.values():
-                lines.append(utf8("Set-Cookie: " + cookie.OutputString(None)))
-        return b"\r\n".join(lines) + b"\r\n\r\n"
+        Requires the `.stream_request_body` decorator.
+        """
+        raise NotImplementedError()
 
     def _log(self):
         """Logs the current request.
@@ -1367,8 +1427,6 @@ def asynchronous(method):
     from tornado.ioloop import IOLoop
     @functools.wraps(method)
     def wrapper(self, *args, **kwargs):
-        if self.application._wsgi:
-            raise Exception("@asynchronous is not supported for WSGI apps")
         self._auto_finish = False
         with stack_context.ExceptionStackContext(
                 self._stack_context_handle_exception):
@@ -1395,6 +1453,40 @@ def asynchronous(method):
     return wrapper
 
 
+def stream_request_body(cls):
+    """Apply to `RequestHandler` subclasses to enable streaming body support.
+
+    This decorator implies the following changes:
+
+    * `.HTTPServerRequest.body` is undefined, and body arguments will not
+      be included in `RequestHandler.get_argument`.
+    * `RequestHandler.prepare` is called when the request headers have been
+      read instead of after the entire body has been read.
+    * The subclass must define a method ``data_received(self, data):``, which
+      will be called zero or more times as data is available.  Note that
+      if the request has an empty body, ``data_received`` may not be called.
+    * ``prepare`` and ``data_received`` may return Futures (such as via
+      ``@gen.coroutine``, in which case the next method will not be called
+      until those futures have completed.
+    * The regular HTTP method (``post``, ``put``, etc) will be called after
+      the entire body has been read.
+
+    There is a subtle interaction between ``data_received`` and asynchronous
+    ``prepare``: The first call to ``data_recieved`` may occur at any point
+    after the call to ``prepare`` has returned *or yielded*.
+    """
+    if not issubclass(cls, RequestHandler):
+        raise TypeError("expected subclass of RequestHandler, got %r", cls)
+    cls._stream_request_body = True
+    return cls
+
+
+def _has_stream_request_body(cls):
+    if not issubclass(cls, RequestHandler):
+        raise TypeError("expected subclass of RequestHandler, got %r", cls)
+    return getattr(cls, '_stream_request_body', False)
+
+
 def removeslash(method):
     """Use this decorator to remove trailing slashes from the request path.
 
@@ -1439,7 +1531,7 @@ def addslash(method):
     return wrapper
 
 
-class Application(object):
+class Application(httputil.HTTPServerConnectionDelegate):
     """A collection of request handlers that make up a web application.
 
     Instances of this class are callable and can be passed directly to
@@ -1491,12 +1583,11 @@ class Application(object):
 
     """
     def __init__(self, handlers=None, default_host="", transforms=None,
-                 wsgi=False, **settings):
+                 **settings):
         if transforms is None:
             self.transforms = []
             if settings.get("gzip"):
                 self.transforms.append(GZipContentEncoding)
-            self.transforms.append(ChunkedTransferEncoding)
         else:
             self.transforms = transforms
         self.handlers = []
@@ -1508,7 +1599,6 @@ class Application(object):
                            'Template': TemplateModule,
                            }
         self.ui_methods = {}
-        self._wsgi = wsgi
         self._load_ui_modules(settings.get("ui_modules", {}))
         self._load_ui_methods(settings.get("ui_methods", {}))
         if self.settings.get("static_path"):
@@ -1534,7 +1624,7 @@ class Application(object):
             self.settings.setdefault('serve_traceback', True)
 
         # Automatically reload modified modules
-        if self.settings.get('autoreload') and not wsgi:
+        if self.settings.get('autoreload'):
             from tornado import autoreload
             autoreload.start()
 
@@ -1634,64 +1724,15 @@ class Application(object):
                 except TypeError:
                     pass
 
+    def start_request(self, connection):
+        # Modern HTTPServer interface
+        return _RequestDispatcher(self, connection)
+
     def __call__(self, request):
-        """Called by HTTPServer to execute the request."""
-        transforms = [t(request) for t in self.transforms]
-        handler = None
-        args = []
-        kwargs = {}
-        handlers = self._get_host_handlers(request)
-        if not handlers:
-            handler = RedirectHandler(
-                self, request, url="http://" + self.default_host + "/")
-        else:
-            for spec in handlers:
-                match = spec.regex.match(request.path)
-                if match:
-                    handler = spec.handler_class(self, request, **spec.kwargs)
-                    if spec.regex.groups:
-                        # None-safe wrapper around url_unescape to handle
-                        # unmatched optional groups correctly
-                        def unquote(s):
-                            if s is None:
-                                return s
-                            return escape.url_unescape(s, encoding=None,
-                                                       plus=False)
-                        # Pass matched groups to the handler.  Since
-                        # match.groups() includes both named and unnamed groups,
-                        # we want to use either groups or groupdict but not both.
-                        # Note that args are passed as bytes so the handler can
-                        # decide what encoding to use.
-
-                        if spec.regex.groupindex:
-                            kwargs = dict(
-                                (str(k), unquote(v))
-                                for (k, v) in match.groupdict().items())
-                        else:
-                            args = [unquote(s) for s in match.groups()]
-                    break
-            if not handler:
-                if self.settings.get('default_handler_class'):
-                    handler_class = self.settings['default_handler_class']
-                    handler_args = self.settings.get(
-                        'default_handler_args', {})
-                else:
-                    handler_class = ErrorHandler
-                    handler_args = dict(status_code=404)
-                handler = handler_class(self, request, **handler_args)
-
-        # If template cache is disabled (usually in the debug mode),
-        # re-compile templates and reload static files on every
-        # request so you don't need to restart to see changes
-        if not self.settings.get("compiled_template_cache", True):
-            with RequestHandler._template_loader_lock:
-                for loader in RequestHandler._template_loaders.values():
-                    loader.reset()
-        if not self.settings.get('static_hash_cache', True):
-            StaticFileHandler.reset()
-
-        handler._execute(transforms, *args, **kwargs)
-        return handler
+        # Legacy HTTPServer interface
+        dispatcher = _RequestDispatcher(self, None)
+        dispatcher.set_request(request)
+        return dispatcher.execute()
 
     def reverse_url(self, name, *args):
         """Returns a URL path for handler named ``name``
@@ -1728,6 +1769,113 @@ class Application(object):
                    handler._request_summary(), request_time)
 
 
+class _RequestDispatcher(httputil.HTTPMessageDelegate):
+    def __init__(self, application, connection):
+        self.application = application
+        self.connection = connection
+        self.request = None
+        self.chunks = []
+        self.handler_class = None
+        self.handler_kwargs = None
+        self.path_args = []
+        self.path_kwargs = {}
+
+    def headers_received(self, start_line, headers):
+        self.set_request(httputil.HTTPServerRequest(
+            connection=self.connection, start_line=start_line, headers=headers))
+        if self.stream_request_body:
+            self.request.body = Future()
+            return self.execute()
+
+    def set_request(self, request):
+        self.request = request
+        self._find_handler()
+        self.stream_request_body = _has_stream_request_body(self.handler_class)
+
+    def _find_handler(self):
+        # Identify the handler to use as soon as we have the request.
+        # Save url path arguments for later.
+        app = self.application
+        handlers = app._get_host_handlers(self.request)
+        if not handlers:
+            self.handler_class = RedirectHandler
+            self.handler_kwargs = dict(url="http://" + app.default_host + "/")
+            return
+        for spec in handlers:
+            match = spec.regex.match(self.request.path)
+            if match:
+                self.handler_class = spec.handler_class
+                self.handler_kwargs = spec.kwargs
+                if spec.regex.groups:
+                    # Pass matched groups to the handler.  Since
+                    # match.groups() includes both named and
+                    # unnamed groups, we want to use either groups
+                    # or groupdict but not both.
+                    if spec.regex.groupindex:
+                        self.path_kwargs = dict(
+                            (str(k), _unquote_or_none(v))
+                            for (k, v) in match.groupdict().items())
+                    else:
+                        self.path_args = [_unquote_or_none(s)
+                                          for s in match.groups()]
+                return
+        if app.settings.get('default_handler_class'):
+            self.handler_class = app.settings['default_handler_class']
+            self.handler_kwargs = app.settings.get(
+                'default_handler_args', {})
+        else:
+            self.handler_class = ErrorHandler
+            self.handler_kwargs = dict(status_code=404)
+
+    def data_received(self, data):
+        if self.stream_request_body:
+            return self.handler.data_received(data)
+        else:
+            self.chunks.append(data)
+
+    def finish(self):
+        if self.stream_request_body:
+            self.request.body.set_result(None)
+        else:
+            self.request.body = b''.join(self.chunks)
+            self.request._parse_body()
+            self.execute()
+
+    def on_connection_close(self):
+        if self.stream_request_body:
+            self.handler.on_connection_close()
+        else:
+            self.chunks = None
+
+    def execute(self):
+        # If template cache is disabled (usually in the debug mode),
+        # re-compile templates and reload static files on every
+        # request so you don't need to restart to see changes
+        if not self.application.settings.get("compiled_template_cache", True):
+            with RequestHandler._template_loader_lock:
+                for loader in RequestHandler._template_loaders.values():
+                    loader.reset()
+        if not self.application.settings.get('static_hash_cache', True):
+            StaticFileHandler.reset()
+
+        self.handler = self.handler_class(self.application, self.request,
+                                          **self.handler_kwargs)
+        transforms = [t(self.request) for t in self.application.transforms]
+
+        if self.stream_request_body:
+            self.handler._prepared_future = Future()
+        # Note that if an exception escapes handler._execute it will be
+        # trapped in the Future it returns (which we are ignoring here).
+        # However, that shouldn't happen because _execute has a blanket
+        # except handler, and we cannot easily access the IOLoop here to
+        # call add_future.
+        self.handler._execute(transforms, *self.path_args, **self.path_kwargs)
+        # If we are streaming the request body, then execute() is finished
+        # when the handler has prepared to receive the body.  If not,
+        # it doesn't matter when execute() finishes (so we return None)
+        return self.handler._prepared_future
+
+
 class HTTPError(Exception):
     """An exception that will turn into an HTTP error response.
 
@@ -1886,8 +2034,9 @@ class StaticFileHandler(RequestHandler):
             cls._static_hashes = {}
 
     def head(self, path):
-        self.get(path, include_body=False)
+        return self.get(path, include_body=False)
 
+    @gen.coroutine
     def get(self, path, include_body=True):
         # Set up our path instance variables.
         self.path = self.parse_url_path(path)
@@ -1912,9 +2061,9 @@ class StaticFileHandler(RequestHandler):
             # the request will be treated as if the header didn't exist.
             request_range = httputil._parse_request_range(range_header)
 
+        size = self.get_content_size()
         if request_range:
             start, end = request_range
-            size = self.get_content_size()
             if (start is not None and start >= size) or end == 0:
                 # As per RFC 2616 14.35.1, a range is not satisfiable only: if
                 # the first requested byte is equal to or greater than the
@@ -1939,18 +2088,26 @@ class StaticFileHandler(RequestHandler):
                                 httputil._get_content_range(start, end, size))
         else:
             start = end = None
-        content = self.get_content(self.absolute_path, start, end)
-        if isinstance(content, bytes_type):
-            content = [content]
-        content_length = 0
-        for chunk in content:
-            if include_body:
+
+        if start is not None and end is not None:
+            content_length = end - start
+        elif end is not None:
+            content_length = end
+        elif start is not None:
+            content_length = size - start
+        else:
+            content_length = size
+        self.set_header("Content-Length", content_length)
+
+        if include_body:
+            content = self.get_content(self.absolute_path, start, end)
+            if isinstance(content, bytes_type):
+                content = [content]
+            for chunk in content:
                 self.write(chunk)
-            else:
-                content_length += len(chunk)
-        if not include_body:
+                yield self.flush()
+        else:
             assert self.request.method == "HEAD"
-            self.set_header("Content-Length", content_length)
 
     def compute_etag(self):
         """Sets the ``Etag`` header based on static url version.
@@ -2130,10 +2287,13 @@ class StaticFileHandler(RequestHandler):
     def get_content_size(self):
         """Retrieve the total size of the resource at the given path.
 
-        This method may be overridden by subclasses. It will only
-        be called if a partial result is requested from `get_content`
+        This method may be overridden by subclasses.
 
         .. versionadded:: 3.1
+
+        .. versionchanged:: 3.3
+           This method is now always called, instead of only when
+           partial results are requested.
         """
         stat_result = self._stat()
         return stat_result[stat.ST_SIZE]
@@ -2255,7 +2415,7 @@ class FallbackHandler(RequestHandler):
     """A `RequestHandler` that wraps another HTTP server callback.
 
     The fallback is a callable object that accepts an
-    `~.httpserver.HTTPRequest`, such as an `Application` or
+    `~.httputil.HTTPServerRequest`, such as an `Application` or
     `tornado.wsgi.WSGIContainer`.  This is most useful to use both
     Tornado ``RequestHandlers`` and WSGI in the same server.  Typical
     usage::
@@ -2279,7 +2439,7 @@ class OutputTransform(object):
     """A transform modifies the result of an HTTP request (e.g., GZip encoding)
 
     A new transform instance is created for every request. See the
-    ChunkedTransferEncoding example below if you want to implement a
+    GZipContentEncoding example below if you want to implement a
     new Transform.
     """
     def __init__(self, request):
@@ -2296,16 +2456,24 @@ class GZipContentEncoding(OutputTransform):
     """Applies the gzip content encoding to the response.
 
     See http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.11
+
+    .. versionchanged:: 3.3
+        Now compresses all mime types beginning with ``text/``, instead
+        of just a whitelist. (the whitelist is still used for certain
+        non-text mime types).
     """
-    CONTENT_TYPES = set([
-        "text/plain", "text/html", "text/css", "text/xml", "application/javascript",
-        "application/x-javascript", "application/xml", "application/atom+xml",
-        "text/javascript", "application/json", "application/xhtml+xml"])
+    # Whitelist of compressible mime types (in addition to any types
+    # beginning with "text/").
+    CONTENT_TYPES = set(["application/javascript", "application/x-javascript",
+                         "application/xml", "application/atom+xml",
+                         "application/json", "application/xhtml+xml"])
     MIN_LENGTH = 5
 
     def __init__(self, request):
-        self._gzipping = request.supports_http_1_1() and \
-            "gzip" in request.headers.get("Accept-Encoding", "")
+        self._gzipping = "gzip" in request.headers.get("Accept-Encoding", "")
+
+    def _compressible_type(self, ctype):
+        return ctype.startswith('text/') or ctype in self.CONTENT_TYPES
 
     def transform_first_chunk(self, status_code, headers, chunk, finishing):
         if 'Vary' in headers:
@@ -2314,7 +2482,7 @@ class GZipContentEncoding(OutputTransform):
             headers['Vary'] = b'Accept-Encoding'
         if self._gzipping:
             ctype = _unicode(headers.get("Content-Type", "")).split(";")[0]
-            self._gzipping = (ctype in self.CONTENT_TYPES) and \
+            self._gzipping = self._compressible_type(ctype) and \
                 (not finishing or len(chunk) >= self.MIN_LENGTH) and \
                 (finishing or "Content-Length" not in headers) and \
                 ("Content-Encoding" not in headers)
@@ -2340,42 +2508,16 @@ class GZipContentEncoding(OutputTransform):
         return chunk
 
 
-class ChunkedTransferEncoding(OutputTransform):
-    """Applies the chunked transfer encoding to the response.
-
-    See http://www.w3.org/Protocols/rfc2616/rfc2616-sec3.html#sec3.6.1
-    """
-    def __init__(self, request):
-        self._chunking = request.supports_http_1_1()
-
-    def transform_first_chunk(self, status_code, headers, chunk, finishing):
-        # 304 responses have no body (not even a zero-length body), and so
-        # should not have either Content-Length or Transfer-Encoding headers.
-        if self._chunking and status_code != 304:
-            # No need to chunk the output if a Content-Length is specified
-            if "Content-Length" in headers or "Transfer-Encoding" in headers:
-                self._chunking = False
-            else:
-                headers["Transfer-Encoding"] = "chunked"
-                chunk = self.transform_chunk(chunk, finishing)
-        return status_code, headers, chunk
-
-    def transform_chunk(self, block, finishing):
-        if self._chunking:
-            # Don't write out empty chunks because that means END-OF-STREAM
-            # with chunked encoding
-            if block:
-                block = utf8("%x" % len(block)) + b"\r\n" + block + b"\r\n"
-            if finishing:
-                block += b"0\r\n\r\n"
-        return block
-
-
 def authenticated(method):
     """Decorate methods with this to require that the user be logged in.
 
     If the user is not logged in, they will be redirected to the configured
     `login url <RequestHandler.get_login_url>`.
+
+    If you configure a login url with a query parameter, Tornado will
+    assume you know what you're doing and use it as-is.  If not, it
+    will add a `next` parameter so the login page knows where to send
+    you once you're logged in.
     """
     @functools.wraps(method)
     def wrapper(self, *args, **kwargs):
@@ -2640,29 +2782,103 @@ else:
         return result == 0
 
 
-def create_signed_value(secret, name, value):
-    timestamp = utf8(str(int(time.time())))
+def create_signed_value(secret, name, value, version=None, clock=None):
+    if version is None:
+        version = DEFAULT_SIGNED_VALUE_VERSION
+    if clock is None:
+        clock = time.time
+    timestamp = utf8(str(int(clock())))
     value = base64.b64encode(utf8(value))
-    signature = _create_signature(secret, name, value, timestamp)
-    value = b"|".join([value, timestamp, signature])
-    return value
+    if version == 1:
+        signature = _create_signature_v1(secret, name, value, timestamp)
+        value = b"|".join([value, timestamp, signature])
+        return value
+    elif version == 2:
+        # The v2 format consists of a version number and a series of
+        # length-prefixed fields "%d:%s", the last of which is a
+        # signature, all separated by pipes.  All numbers are in
+        # decimal format with no leading zeros.  The signature is an
+        # HMAC-SHA256 of the whole string up to that point, including
+        # the final pipe.
+        #
+        # The fields are:
+        # - format version (i.e. 2; no length prefix)
+        # - key version (currently 0; reserved for future key rotation features)
+        # - timestamp (integer seconds since epoch)
+        # - name (not encoded; assumed to be ~alphanumeric)
+        # - value (base64-encoded)
+        # - signature (hex-encoded; no length prefix)
+        def format_field(s):
+            return utf8("%d:" % len(s)) + utf8(s)
+        to_sign = b"|".join([
+            b"2|1:0",
+            format_field(timestamp),
+            format_field(name),
+            format_field(value),
+            b''])
+        signature = _create_signature_v2(secret, to_sign)
+        return to_sign + signature
+    else:
+        raise ValueError("Unsupported version %d" % version)
+
+# A leading version number in decimal with no leading zeros, followed by a pipe.
+_signed_value_version_re = re.compile(br"^([1-9][0-9]*)\|(.*)$")
 
 
-def decode_signed_value(secret, name, value, max_age_days=31):
+def decode_signed_value(secret, name, value, max_age_days=31, clock=None, min_version=None):
+    if clock is None:
+        clock = time.time
+    if min_version is None:
+        min_version = DEFAULT_SIGNED_VALUE_MIN_VERSION
+    if min_version > 2:
+        raise ValueError("Unsupported min_version %d" % min_version)
     if not value:
         return None
+
+    # Figure out what version this is.  Version 1 did not include an
+    # explicit version field and started with arbitrary base64 data,
+    # which makes this tricky.
+    value = utf8(value)
+    m = _signed_value_version_re.match(value)
+    if m is None:
+        version = 1
+    else:
+        try:
+            version = int(m.group(1))
+            if version > 999:
+                # Certain payloads from the version-less v1 format may
+                # be parsed as valid integers.  Due to base64 padding
+                # restrictions, this can only happen for numbers whose
+                # length is a multiple of 4, so we can treat all
+                # numbers up to 999 as versions, and for the rest we
+                # fall back to v1 format.
+                version = 1
+        except ValueError:
+            version = 1
+
+    if version < min_version:
+        return None
+    if version == 1:
+        return _decode_signed_value_v1(secret, name, value, max_age_days, clock)
+    elif version == 2:
+        return _decode_signed_value_v2(secret, name, value, max_age_days, clock)
+    else:
+        return None
+
+
+def _decode_signed_value_v1(secret, name, value, max_age_days, clock):
     parts = utf8(value).split(b"|")
     if len(parts) != 3:
         return None
-    signature = _create_signature(secret, name, parts[0], parts[1])
+    signature = _create_signature_v1(secret, name, parts[0], parts[1])
     if not _time_independent_equals(parts[2], signature):
         gen_log.warning("Invalid cookie signature %r", value)
         return None
     timestamp = int(parts[1])
-    if timestamp < time.time() - max_age_days * 86400:
+    if timestamp < clock() - max_age_days * 86400:
         gen_log.warning("Expired cookie %r", value)
         return None
-    if timestamp > time.time() + 31 * 86400:
+    if timestamp > clock() + 31 * 86400:
         # _cookie_signature does not hash a delimiter between the
         # parts of the cookie, so an attacker could transfer trailing
         # digits from the payload to the timestamp without altering the
@@ -2679,8 +2895,62 @@ def decode_signed_value(secret, name, value, max_age_days=31):
         return None
 
 
-def _create_signature(secret, *parts):
+def _decode_signed_value_v2(secret, name, value, max_age_days, clock):
+    def _consume_field(s):
+        length, _, rest = s.partition(b':')
+        n = int(length)
+        field_value = rest[:n]
+        # In python 3, indexing bytes returns small integers; we must
+        # use a slice to get a byte string as in python 2.
+        if rest[n:n + 1] != b'|':
+            raise ValueError("malformed v2 signed value field")
+        rest = rest[n + 1:]
+        return field_value, rest
+    rest = value[2:]  # remove version number
+    try:
+        key_version, rest = _consume_field(rest)
+        timestamp, rest = _consume_field(rest)
+        name_field, rest = _consume_field(rest)
+        value_field, rest = _consume_field(rest)
+    except ValueError:
+        return None
+    passed_sig = rest
+    signed_string = value[:-len(passed_sig)]
+    expected_sig = _create_signature_v2(secret, signed_string)
+    if not _time_independent_equals(passed_sig, expected_sig):
+        return None
+    if name_field != utf8(name):
+        return None
+    timestamp = int(timestamp)
+    if timestamp < clock() - max_age_days * 86400:
+        # The signature has expired.
+        return None
+    try:
+        return base64.b64decode(value_field)
+    except Exception:
+        return None
+
+
+def _create_signature_v1(secret, *parts):
     hash = hmac.new(utf8(secret), digestmod=hashlib.sha1)
     for part in parts:
         hash.update(utf8(part))
     return utf8(hash.hexdigest())
+
+
+def _create_signature_v2(secret, s):
+    hash = hmac.new(utf8(secret), digestmod=hashlib.sha256)
+    hash.update(utf8(s))
+    return utf8(hash.hexdigest())
+
+
+def _unquote_or_none(s):
+    """None-safe wrapper around url_unescape to handle unamteched optional
+    groups correctly.
+
+    Note that args are passed as bytes so the handler can decide what
+    encoding to use.
+    """
+    if s is None:
+        return s
+    return escape.url_unescape(s, encoding=None, plus=False)