Update Tornado

libs/tornado/web.py

@@ -52,6 +52,7 @@ request.
 
 from __future__ import absolute_import, division, print_function, with_statement
 
+
 import base64
 import binascii
 import datetime
@@ -131,14 +132,14 @@ class RequestHandler(object):
         self.path_kwargs = None
         self.ui = ObjectDict((n, self._ui_method(m)) for n, m in
                              application.ui_methods.items())
-        # UIModules are available as both `modules` and `_modules` in the
+        # UIModules are available as both `modules` and `_tt_modules` in the
         # template namespace.  Historically only `modules` was available
         # but could be clobbered by user additions to the namespace.
-        # The template {% module %} directive looks in `_modules` to avoid
+        # The template {% module %} directive looks in `_tt_modules` to avoid
         # possible conflicts.
-        self.ui["_modules"] = ObjectDict((n, self._ui_module(n, m)) for n, m in
-                                         application.ui_modules.items())
-        self.ui["modules"] = self.ui["_modules"]
+        self.ui["_tt_modules"] = _UIModuleNamespace(self,
+                                                    application.ui_modules)
+        self.ui["modules"] = self.ui["_tt_modules"]
         self.clear()
         # Check since connection is not available in WSGI
         if getattr(self.request, "connection", None):
@@ -198,6 +199,15 @@ class RequestHandler(object):
 
         Override this method to perform common initialization regardless
         of the request method.
+
+        Asynchronous support: Decorate this method with `.gen.coroutine`
+        or `.return_future` to make it asynchronous (the
+        `asynchronous` decorator cannot be used on `prepare`).
+        If this method returns a `.Future` execution will not proceed
+        until the `.Future` is done.
+
+        .. versionadded:: 3.1
+           Asynchronous support.
         """
         pass
 
@@ -232,11 +242,14 @@ class RequestHandler(object):
         self._headers = httputil.HTTPHeaders({
             "Server": "TornadoServer/%s" % tornado.version,
             "Content-Type": "text/html; charset=UTF-8",
-            "Date": httputil.format_timestamp(time.gmtime()),
+            "Date": httputil.format_timestamp(time.time()),
         })
         self.set_default_headers()
-        if not self.request.supports_http_1_1():
-            if self.request.headers.get("Connection") == "Keep-Alive":
+        if (not self.request.supports_http_1_1() and
+            getattr(self.request, 'connection', None) and
+                not self.request.connection.no_keep_alive):
+            conn_header = self.request.headers.get("Connection")
+            if conn_header and (conn_header.lower() == "keep-alive"):
                 self.set_header("Connection", "Keep-Alive")
         self._write_buffer = []
         self._status_code = 200
@@ -300,6 +313,8 @@ class RequestHandler(object):
         if name in self._headers:
             del self._headers[name]
 
+    _INVALID_HEADER_CHAR_RE = re.compile(br"[\x00-\x1f]")
+
     def _convert_header_value(self, value):
         if isinstance(value, bytes_type):
             pass
@@ -315,7 +330,8 @@ class RequestHandler(object):
         # If \n is allowed into the header, it is possible to inject
         # additional headers or split the request. Also cap length to
         # prevent obviously erroneous values.
-        if len(value) > 4000 or re.search(br"[\x00-\x1f]", value):
+        if (len(value) > 4000 or
+                RequestHandler._INVALID_HEADER_CHAR_RE.search(value)):
             raise ValueError("Unsafe header value %r", value)
         return value
 
@@ -325,7 +341,7 @@ class RequestHandler(object):
         """Returns the value of the argument with the given name.
 
         If default is not provided, the argument is considered to be
-        required, and we throw an HTTP 400 exception if it is missing.
+        required, and we raise a `MissingArgumentError` if it is missing.
 
         If the argument appears in the url more than once, we return the
         last value.
@@ -335,7 +351,7 @@ class RequestHandler(object):
         args = self.get_arguments(name, strip=strip)
         if not args:
             if default is self._ARG_DEFAULT:
-                raise HTTPError(400, "Missing argument %s" % name)
+                raise MissingArgumentError(name)
             return default
         return args[-1]
 
@@ -488,10 +504,8 @@ class RequestHandler(object):
         else:
             assert isinstance(status, int) and 300 <= status <= 399
         self.set_status(status)
-        # Remove whitespace
-        url = re.sub(br"[\x00-\x20]+", "", utf8(url))
         self.set_header("Location", urlparse.urljoin(utf8(self.request.uri),
-                                                     url))
+                                                     utf8(url)))
         self.finish()
 
     def write(self, chunk):
@@ -680,7 +694,11 @@ class RequestHandler(object):
         has been run, the previous callback will be discarded.
         """
         if self.application._wsgi:
-            raise Exception("WSGI applications do not support flush()")
+            # WSGI applications cannot usefully support flush, so just make
+            # it a no-op (and run the callback immediately).
+            if callback is not None:
+                callback()
+            return
 
         chunk = b"".join(self._write_buffer)
         self._write_buffer = []
@@ -720,13 +738,10 @@ class RequestHandler(object):
             if (self._status_code == 200 and
                 self.request.method in ("GET", "HEAD") and
                     "Etag" not in self._headers):
-                etag = self.compute_etag()
-                if etag is not None:
-                    self.set_header("Etag", etag)
-                    inm = self.request.headers.get("If-None-Match")
-                    if inm and inm.find(etag) != -1:
-                        self._write_buffer = []
-                        self.set_status(304)
+                self.set_etag_header()
+                if self.check_etag_header():
+                    self._write_buffer = []
+                    self.set_status(304)
             if self._status_code == 304:
                 assert not self._write_buffer, "Cannot send body with 304"
                 self._clear_headers_for_304()
@@ -891,6 +906,10 @@ class RequestHandler(object):
             self._current_user = self.get_current_user()
         return self._current_user
 
+    @current_user.setter
+    def current_user(self, value):
+        self._current_user = value
+
     def get_current_user(self):
         """Override to determine the current user from, e.g., a cookie."""
         return None
@@ -976,27 +995,30 @@ class RequestHandler(object):
         return '<input type="hidden" name="_xsrf" value="' + \
             escape.xhtml_escape(self.xsrf_token) + '"/>'
 
-    def static_url(self, path, include_host=None):
+    def static_url(self, path, include_host=None, **kwargs):
         """Returns a static URL for the given relative static file path.
 
         This method requires you set the ``static_path`` setting in your
         application (which specifies the root directory of your static
         files).
 
-        We append ``?v=<signature>`` to the returned URL, which makes our
-        static file handler set an infinite expiration header on the
-        returned content. The signature is based on the content of the
-        file.
+        This method returns a versioned url (by default appending
+        ``?v=<signature>``), which allows the static files to be
+        cached indefinitely.  This can be disabled by passing
+        ``include_version=False`` (in the default implementation;
+        other static file implementations are not required to support
+        this, but they may support other options).
 
         By default this method returns URLs relative to the current
         host, but if ``include_host`` is true the URL returned will be
         absolute.  If this handler has an ``include_host`` attribute,
         that value will be used as the default for all `static_url`
         calls that do not pass ``include_host`` as a keyword argument.
+
         """
         self.require_setting("static_path", "static_url")
-        static_handler_class = self.settings.get(
-            "static_handler_class", StaticFileHandler)
+        get_url = self.settings.get("static_handler_class",
+                                    StaticFileHandler).make_static_url
 
         if include_host is None:
             include_host = getattr(self, "include_host", False)
@@ -1005,7 +1027,8 @@ class RequestHandler(object):
             base = self.request.protocol + "://" + self.request.host
         else:
             base = ""
-        return base + static_handler_class.make_static_url(self.settings, path)
+
+        return base + get_url(self.settings, path, **kwargs)
 
     def async_callback(self, callback, *args, **kwargs):
         """Obsolete - catches exceptions from the wrapped function.
@@ -1041,6 +1064,8 @@ class RequestHandler(object):
     def compute_etag(self):
         """Computes the etag header to be used for this request.
 
+        By default uses a hash of the content written so far.
+
         May be overridden to provide custom etag implementations,
         or may return None to disable tornado's default etag support.
         """
@@ -1049,6 +1074,38 @@ class RequestHandler(object):
             hasher.update(part)
         return '"%s"' % hasher.hexdigest()
 
+    def set_etag_header(self):
+        """Sets the response's Etag header using ``self.compute_etag()``.
+
+        Note: no header will be set if ``compute_etag()`` returns ``None``.
+
+        This method is called automatically when the request is finished.
+        """
+        etag = self.compute_etag()
+        if etag is not None:
+            self.set_header("Etag", etag)
+
+    def check_etag_header(self):
+        """Checks the ``Etag`` header against requests's ``If-None-Match``.
+
+        Returns ``True`` if the request's Etag matches and a 304 should be
+        returned. For example::
+
+            self.set_etag_header()
+            if self.check_etag_header():
+                self.set_status(304)
+                return
+
+        This method is called automatically when the request is finished,
+        but may be called earlier for applications that override
+        `compute_etag` and want to do an early check for ``If-None-Match``
+        before completing the request.  The ``Etag`` header should be set
+        (perhaps with `set_etag_header`) before calling this method.
+        """
+        etag = self._headers.get("Etag")
+        inm = utf8(self.request.headers.get("If-None-Match", ""))
+        return bool(etag and inm and inm.find(etag) >= 0)
+
     def _stack_context_handle_exception(self, type, value, traceback):
         try:
             # For historical reasons _handle_request_exception only takes
@@ -1074,15 +1131,41 @@ class RequestHandler(object):
             if self.request.method not in ("GET", "HEAD", "OPTIONS") and \
                     self.application.settings.get("xsrf_cookies"):
                 self.check_xsrf_cookie()
-            self.prepare()
-            if not self._finished:
-                getattr(self, self.request.method.lower())(
-                    *self.path_args, **self.path_kwargs)
-                if self._auto_finish and not self._finished:
-                    self.finish()
+            self._when_complete(self.prepare(), self._execute_method)
         except Exception as e:
             self._handle_request_exception(e)
 
+    def _when_complete(self, result, callback):
+        try:
+            if result is None:
+                callback()
+            elif isinstance(result, Future):
+                if result.done():
+                    if result.result() is not None:
+                        raise ValueError('Expected None, got %r' % result)
+                    callback()
+                else:
+                    # Delayed import of IOLoop because it's not available
+                    # on app engine
+                    from tornado.ioloop import IOLoop
+                    IOLoop.current().add_future(
+                        result, functools.partial(self._when_complete,
+                                                  callback=callback))
+            else:
+                raise ValueError("Expected Future or None, got %r" % result)
+        except Exception as e:
+            self._handle_request_exception(e)
+
+    def _execute_method(self):
+        if not self._finished:
+            method = getattr(self, self.request.method.lower())
+            self._when_complete(method(*self.path_args, **self.path_kwargs),
+                                self._execute_finish)
+
+    def _execute_finish(self):
+        if self._auto_finish and not self._finished:
+            self.finish()
+
     def _generate_headers(self):
         reason = self._reason
         lines = [utf8(self.request.version + " " +
@@ -1109,21 +1192,41 @@ class RequestHandler(object):
             " (" + self.request.remote_ip + ")"
 
     def _handle_request_exception(self, e):
+        self.log_exception(*sys.exc_info())
+        if self._finished:
+            # Extra errors after the request has been finished should
+            # be logged, but there is no reason to continue to try and
+            # send a response.
+            return
         if isinstance(e, HTTPError):
-            if e.log_message:
-                format = "%d %s: " + e.log_message
-                args = [e.status_code, self._request_summary()] + list(e.args)
-                gen_log.warning(format, *args)
             if e.status_code not in httputil.responses and not e.reason:
                 gen_log.error("Bad HTTP status code: %d", e.status_code)
                 self.send_error(500, exc_info=sys.exc_info())
             else:
                 self.send_error(e.status_code, exc_info=sys.exc_info())
         else:
-            app_log.error("Uncaught exception %s\n%r", self._request_summary(),
-                          self.request, exc_info=True)
             self.send_error(500, exc_info=sys.exc_info())
 
+    def log_exception(self, typ, value, tb):
+        """Override to customize logging of uncaught exceptions.
+
+        By default logs instances of `HTTPError` as warnings without
+        stack traces (on the ``tornado.general`` logger), and all
+        other exceptions as errors with stack traces (on the
+        ``tornado.application`` logger).
+
+        .. versionadded:: 3.1
+        """
+        if isinstance(value, HTTPError):
+            if value.log_message:
+                format = "%d %s: " + value.log_message
+                args = ([value.status_code, self._request_summary()] +
+                        list(value.args))
+                gen_log.warning(format, *args)
+        else:
+            app_log.error("Uncaught exception %s\n%r", self._request_summary(),
+                          self.request, exc_info=(typ, value, tb))
+
     def _ui_module(self, name, module):
         def render(*args, **kwargs):
             if not hasattr(self, "_active_modules"):
@@ -1152,6 +1255,11 @@ class RequestHandler(object):
 def asynchronous(method):
     """Wrap request handler methods with this if they are asynchronous.
 
+    This decorator is unnecessary if the method is also decorated with
+    ``@gen.coroutine`` (it is legal but unnecessary to use the two
+    decorators together, in which case ``@asynchronous`` must be
+    first).
+
     This decorator should only be applied to the :ref:`HTTP verb
     methods <verbs>`; its behavior is undefined for any other method.
     This decorator does not *make* a method asynchronous; it tells
@@ -1175,6 +1283,8 @@ def asynchronous(method):
               self.write("Downloaded!")
               self.finish()
 
+    .. versionadded:: 3.1
+       The ability to use ``@gen.coroutine`` without ``@asynchronous``.
     """
     # Delay the IOLoop import because it's not available on app engine.
     from tornado.ioloop import IOLoop
@@ -1461,7 +1571,8 @@ class Application(object):
                         def unquote(s):
                             if s is None:
                                 return s
-                            return escape.url_unescape(s, encoding=None)
+                            return escape.url_unescape(s, encoding=None,
+                                                       plus=False)
                         # Pass matched groups to the handler.  Since
                         # match.groups() includes both named and unnamed groups,
                         # we want to use either groups or groupdict but not both.
@@ -1559,6 +1670,20 @@ class HTTPError(Exception):
             return message
 
 
+class MissingArgumentError(HTTPError):
+    """Exception raised by `RequestHandler.get_argument`.
+
+    This is a subclass of `HTTPError`, so if it is uncaught a 400 response
+    code will be used instead of 500 (and a stack trace will not be logged).
+
+    .. versionadded:: 3.1
+    """
+    def __init__(self, arg_name):
+        super(MissingArgumentError, self).__init__(
+            400, 'Missing argument %s' % arg_name)
+        self.arg_name = arg_name
+
+
 class ErrorHandler(RequestHandler):
     """Generates an error response with ``status_code`` for all requests."""
     def initialize(self, status_code):
@@ -1594,21 +1719,59 @@ class RedirectHandler(RequestHandler):
 class StaticFileHandler(RequestHandler):
     """A simple handler that can serve static content from a directory.
 
-    To map a path to this handler for a static data directory ``/var/www``,
+    A `StaticFileHandler` is configured automatically if you pass the
+    ``static_path`` keyword argument to `Application`.  This handler
+    can be customized with the ``static_url_prefix``, ``static_handler_class``,
+    and ``static_handler_args`` settings.
+
+    To map an additional path to this handler for a static data directory
     you would add a line to your application like::
 
         application = web.Application([
-            (r"/static/(.*)", web.StaticFileHandler, {"path": "/var/www"}),
+            (r"/content/(.*)", web.StaticFileHandler, {"path": "/var/www"}),
         ])
 
-    The local root directory of the content should be passed as the ``path``
-    argument to the handler.
+    The handler constructor requires a ``path`` argument, which specifies the
+    local root directory of the content to be served.
 
-    To support aggressive browser caching, if the argument ``v`` is given
-    with the path, we set an infinite HTTP expiration header. So, if you
-    want browsers to cache a file indefinitely, send them to, e.g.,
-    ``/static/images/myimage.png?v=xxx``. Override `get_cache_time` method for
-    more fine-grained cache control.
+    Note that a capture group in the regex is required to parse the value for
+    the ``path`` argument to the get() method (different than the constructor
+    argument above); see `URLSpec` for details.
+
+    To maximize the effectiveness of browser caching, this class supports
+    versioned urls (by default using the argument ``?v=``).  If a version
+    is given, we instruct the browser to cache this file indefinitely.
+    `make_static_url` (also available as `RequestHandler.static_url`) can
+    be used to construct a versioned url.
+
+    This handler is intended primarily for use in development and light-duty
+    file serving; for heavy traffic it will be more efficient to use
+    a dedicated static file server (such as nginx or Apache).  We support
+    the HTTP ``Accept-Ranges`` mechanism to return partial content (because
+    some browsers require this functionality to be present to seek in
+    HTML5 audio or video), but this handler should not be used with
+    files that are too large to fit comfortably in memory.
+
+    **Subclassing notes**
+
+    This class is designed to be extensible by subclassing, but because
+    of the way static urls are generated with class methods rather than
+    instance methods, the inheritance patterns are somewhat unusual.
+    Be sure to use the ``@classmethod`` decorator when overriding a
+    class method.  Instance methods may use the attributes ``self.path``
+    ``self.absolute_path``, and ``self.modified``.
+
+    To change the way static urls are generated (e.g. to match the behavior
+    of another server or CDN), override `make_static_url`, `parse_url_path`,
+    `get_cache_time`, and/or `get_version`.
+
+    To replace all interaction with the filesystem (e.g. to serve
+    static content from a database), override `get_content`,
+    `get_content_size`, `get_modified_time`, `get_absolute_path`, and
+    `validate_absolute_path`.
+
+    .. versionchanged:: 3.1
+       Many of the methods for subclasses were added in Tornado 3.1.
     """
     CACHE_MAX_AGE = 86400 * 365 * 10  # 10 years
 
@@ -1616,7 +1779,7 @@ class StaticFileHandler(RequestHandler):
     _lock = threading.Lock()  # protects _static_hashes
 
     def initialize(self, path, default_filename=None):
-        self.root = os.path.abspath(path) + os.path.sep
+        self.root = path
         self.default_filename = default_filename
 
     @classmethod
@@ -1628,60 +1791,270 @@ class StaticFileHandler(RequestHandler):
         self.get(path, include_body=False)
 
     def get(self, path, include_body=True):
-        path = self.parse_url_path(path)
-        abspath = os.path.abspath(os.path.join(self.root, path))
-        # os.path.abspath strips a trailing /
-        # it needs to be temporarily added back for requests to root/
-        if not (abspath + os.path.sep).startswith(self.root):
-            raise HTTPError(403, "%s is not in root static directory", path)
-        if os.path.isdir(abspath) and self.default_filename is not None:
-            # need to look at the request.path here for when path is empty
-            # but there is some prefix to the path that was already
-            # trimmed by the routing
-            if not self.request.path.endswith("/"):
-                self.redirect(self.request.path + "/")
+        # Set up our path instance variables.
+        self.path = self.parse_url_path(path)
+        del path  # make sure we don't refer to path instead of self.path again
+        absolute_path = self.get_absolute_path(self.root, self.path)
+        self.absolute_path = self.validate_absolute_path(
+            self.root, absolute_path)
+        if self.absolute_path is None:
+            return
+
+        self.modified = self.get_modified_time()
+        self.set_headers()
+
+        if self.should_return_304():
+            self.set_status(304)
+            return
+
+        request_range = None
+        range_header = self.request.headers.get("Range")
+        if range_header:
+            # As per RFC 2616 14.16, if an invalid Range header is specified,
+            # the request will be treated as if the header didn't exist.
+            request_range = httputil._parse_request_range(range_header)
+
+        if request_range:
+            start, end = request_range
+            size = self.get_content_size()
+            if (start is not None and start >= size) or end == 0:
+                # As per RFC 2616 14.35.1, a range is not satisfiable only: if
+                # the first requested byte is equal to or greater than the
+                # content, or when a suffix with length 0 is specified
+                self.set_status(416)  # Range Not Satisfiable
+                self.set_header("Content-Type", "text/plain")
+                self.set_header("Content-Range", "bytes */%s" %(size, ))
                 return
-            abspath = os.path.join(abspath, self.default_filename)
-        if not os.path.exists(abspath):
-            raise HTTPError(404)
-        if not os.path.isfile(abspath):
-            raise HTTPError(403, "%s is not a file", path)
+            if start is not None and start < 0:
+                start += size
+            # Note: only return HTTP 206 if less than the entire range has been
+            # requested. Not only is this semantically correct, but Chrome
+            # refuses to play audio if it gets an HTTP 206 in response to
+            # ``Range: bytes=0-``.
+            if size != (end or size) - (start or 0):
+                self.set_status(206)  # Partial Content
+                self.set_header("Content-Range",
+                                httputil._get_content_range(start, end, size))
+        else:
+            start = end = None
+        content = self.get_content(self.absolute_path, start, end)
+        if isinstance(content, bytes_type):
+            content = [content]
+        content_length = 0
+        for chunk in content:
+            if include_body:
+                self.write(chunk)
+            else:
+                content_length += len(chunk)
+        if not include_body:
+            assert self.request.method == "HEAD"
+            self.set_header("Content-Length", content_length)
 
-        stat_result = os.stat(abspath)
-        modified = datetime.datetime.utcfromtimestamp(stat_result[stat.ST_MTIME])
+    def compute_etag(self):
+        """Sets the ``Etag`` header based on static url version.
 
-        self.set_header("Last-Modified", modified)
+        This allows efficient ``If-None-Match`` checks against cached
+        versions, and sends the correct ``Etag`` for a partial response
+        (i.e. the same ``Etag`` as the full file).
 
-        mime_type, encoding = mimetypes.guess_type(abspath)
-        if mime_type:
-            self.set_header("Content-Type", mime_type)
+        .. versionadded:: 3.1
+        """
+        version_hash = self._get_cached_version(self.absolute_path)
+        if not version_hash:
+            return None
+        return '"%s"' % (version_hash, )
 
-        cache_time = self.get_cache_time(path, modified, mime_type)
+    def set_headers(self):
+        """Sets the content and caching headers on the response.
 
+        .. versionadded:: 3.1
+        """
+        self.set_header("Accept-Ranges", "bytes")
+        self.set_etag_header()
+
+        if self.modified is not None:
+            self.set_header("Last-Modified", self.modified)
+
+        content_type = self.get_content_type()
+        if content_type:
+            self.set_header("Content-Type", content_type)
+
+        cache_time = self.get_cache_time(self.path, self.modified, content_type)
         if cache_time > 0:
             self.set_header("Expires", datetime.datetime.utcnow() +
                             datetime.timedelta(seconds=cache_time))
             self.set_header("Cache-Control", "max-age=" + str(cache_time))
 
-        self.set_extra_headers(path)
+        self.set_extra_headers(self.path)
+
+    def should_return_304(self):
+        """Returns True if the headers indicate that we should return 304.
+
+        .. versionadded:: 3.1
+        """
+        if self.check_etag_header():
+            return True
 
         # Check the If-Modified-Since, and don't send the result if the
         # content has not been modified
         ims_value = self.request.headers.get("If-Modified-Since")
         if ims_value is not None:
             date_tuple = email.utils.parsedate(ims_value)
-            if_since = datetime.datetime(*date_tuple[:6])
-            if if_since >= modified:
-                self.set_status(304)
-                return
+            if date_tuple is not None:
+                if_since = datetime.datetime(*date_tuple[:6])
+                if if_since >= self.modified:
+                    return True
 
+        return False
+
+    @classmethod
+    def get_absolute_path(cls, root, path):
+        """Returns the absolute location of ``path`` relative to ``root``.
+
+        ``root`` is the path configured for this `StaticFileHandler`
+        (in most cases the ``static_path`` `Application` setting).
+
+        This class method may be overridden in subclasses.  By default
+        it returns a filesystem path, but other strings may be used
+        as long as they are unique and understood by the subclass's
+        overridden `get_content`.
+
+        .. versionadded:: 3.1
+        """
+        abspath = os.path.abspath(os.path.join(root, path))
+        return abspath
+
+    def validate_absolute_path(self, root, absolute_path):
+        """Validate and return the absolute path.
+
+        ``root`` is the configured path for the `StaticFileHandler`,
+        and ``path`` is the result of `get_absolute_path`
+
+        This is an instance method called during request processing,
+        so it may raise `HTTPError` or use methods like
+        `RequestHandler.redirect` (return None after redirecting to
+        halt further processing).  This is where 404 errors for missing files
+        are generated.
+
+        This method may modify the path before returning it, but note that
+        any such modifications will not be understood by `make_static_url`.
+
+        In instance methods, this method's result is available as
+        ``self.absolute_path``.
+
+        .. versionadded:: 3.1
+        """
+        root = os.path.abspath(root)
+        # os.path.abspath strips a trailing /
+        # it needs to be temporarily added back for requests to root/
+        if not (absolute_path + os.path.sep).startswith(root):
+            raise HTTPError(403, "%s is not in root static directory",
+                            self.path)
+        if (os.path.isdir(absolute_path) and
+                self.default_filename is not None):
+            # need to look at the request.path here for when path is empty
+            # but there is some prefix to the path that was already
+            # trimmed by the routing
+            if not self.request.path.endswith("/"):
+                self.redirect(self.request.path + "/", permanent=True)
+                return
+            absolute_path = os.path.join(absolute_path, self.default_filename)
+        if not os.path.exists(absolute_path):
+            raise HTTPError(404)
+        if not os.path.isfile(absolute_path):
+            raise HTTPError(403, "%s is not a file", self.path)
+        return absolute_path
+
+    @classmethod
+    def get_content(cls, abspath, start=None, end=None):
+        """Retrieve the content of the requested resource which is located
+        at the given absolute path.
+
+        This class method may be overridden by subclasses.  Note that its
+        signature is different from other overridable class methods
+        (no ``settings`` argument); this is deliberate to ensure that
+        ``abspath`` is able to stand on its own as a cache key.
+
+        This method should either return a byte string or an iterator
+        of byte strings.  The latter is preferred for large files
+        as it helps reduce memory fragmentation.
+
+        .. versionadded:: 3.1
+        """
         with open(abspath, "rb") as file:
-            data = file.read()
-            if include_body:
-                self.write(data)
+            if start is not None:
+                file.seek(start)
+            if end is not None:
+                remaining = end - (start or 0)
             else:
-                assert self.request.method == "HEAD"
-                self.set_header("Content-Length", len(data))
+                remaining = None
+            while True:
+                chunk_size = 64 * 1024
+                if remaining is not None and remaining < chunk_size:
+                    chunk_size = remaining
+                chunk = file.read(chunk_size)
+                if chunk:
+                    if remaining is not None:
+                        remaining -= len(chunk)
+                    yield chunk
+                else:
+                    if remaining is not None:
+                        assert remaining == 0
+                    return
+
+    @classmethod
+    def get_content_version(cls, abspath):
+        """Returns a version string for the resource at the given path.
+
+        This class method may be overridden by subclasses.  The
+        default implementation is a hash of the file's contents.
+
+        .. versionadded:: 3.1
+        """
+        data = cls.get_content(abspath)
+        hasher = hashlib.md5()
+        if isinstance(data, bytes_type):
+            hasher.update(data)
+        else:
+            for chunk in data:
+                hasher.update(chunk)
+        return hasher.hexdigest()
+
+    def _stat(self):
+        if not hasattr(self, '_stat_result'):
+            self._stat_result = os.stat(self.absolute_path)
+        return self._stat_result
+
+    def get_content_size(self):
+        """Retrieve the total size of the resource at the given path.
+
+        This method may be overridden by subclasses. It will only
+        be called if a partial result is requested from `get_content`
+
+        .. versionadded:: 3.1
+        """
+        stat_result = self._stat()
+        return stat_result[stat.ST_SIZE]
+
+    def get_modified_time(self):
+        """Returns the time that ``self.absolute_path`` was last modified.
+
+        May be overridden in subclasses.  Should return a `~datetime.datetime`
+        object or None.
+
+        .. versionadded:: 3.1
+        """
+        stat_result = self._stat()
+        modified = datetime.datetime.utcfromtimestamp(stat_result[stat.ST_MTIME])
+        return modified
+
+    def get_content_type(self):
+        """Returns the ``Content-Type`` header to be used for this request.
+
+        .. versionadded:: 3.1
+        """
+        mime_type, encoding = mimetypes.guess_type(self.absolute_path)
+        return mime_type
 
     def set_extra_headers(self, path):
         """For subclass to add extra headers to the response"""
@@ -1701,50 +2074,34 @@ class StaticFileHandler(RequestHandler):
         return self.CACHE_MAX_AGE if "v" in self.request.arguments else 0
 
     @classmethod
-    def make_static_url(cls, settings, path):
+    def make_static_url(cls, settings, path, include_version=True):
         """Constructs a versioned url for the given path.
 
-        This method may be overridden in subclasses (but note that it is
-        a class method rather than an instance method).
+        This method may be overridden in subclasses (but note that it
+        is a class method rather than an instance method).  Subclasses
+        are only required to implement the signature
+        ``make_static_url(cls, settings, path)``; other keyword
+        arguments may be passed through `~RequestHandler.static_url`
+        but are not standard.
 
         ``settings`` is the `Application.settings` dictionary.  ``path``
         is the static path being requested.  The url returned should be
         relative to the current host.
+
+        ``include_version`` determines whether the generated URL should
+        include the query string containing the version hash of the
+        file corresponding to the given ``path``.
+
         """
-        static_url_prefix = settings.get('static_url_prefix', '/static/')
+        url = settings.get('static_url_prefix', '/static/') + path
+        if not include_version:
+            return url
+
         version_hash = cls.get_version(settings, path)
-        if version_hash:
-            return static_url_prefix + path + "?v=" + version_hash
-        return static_url_prefix + path
+        if not version_hash:
+            return url
 
-    @classmethod
-    def get_version(cls, settings, path):
-        """Generate the version string to be used in static URLs.
-
-        This method may be overridden in subclasses (but note that it
-        is a class method rather than a static method).  The default
-        implementation uses a hash of the file's contents.
-
-        ``settings`` is the `Application.settings` dictionary and ``path``
-        is the relative location of the requested asset on the filesystem.
-        The returned value should be a string, or ``None`` if no version
-        could be determined.
-        """
-        abs_path = os.path.join(settings["static_path"], path)
-        with cls._lock:
-            hashes = cls._static_hashes
-            if abs_path not in hashes:
-                try:
-                    f = open(abs_path, "rb")
-                    hashes[abs_path] = hashlib.md5(f.read()).hexdigest()
-                    f.close()
-                except Exception:
-                    gen_log.error("Could not open static file %r", path)
-                    hashes[abs_path] = None
-            hsh = hashes.get(abs_path)
-            if hsh:
-                return hsh[:5]
-        return None
+        return '%s?v=%s' % (url, version_hash)
 
     def parse_url_path(self, url_path):
         """Converts a static URL path into a filesystem path.
@@ -1752,11 +2109,45 @@ class StaticFileHandler(RequestHandler):
         ``url_path`` is the path component of the URL with
         ``static_url_prefix`` removed.  The return value should be
         filesystem path relative to ``static_path``.
+
+        This is the inverse of `make_static_url`.
         """
         if os.path.sep != "/":
             url_path = url_path.replace("/", os.path.sep)
         return url_path
 
+    @classmethod
+    def get_version(cls, settings, path):
+        """Generate the version string to be used in static URLs.
+
+        ``settings`` is the `Application.settings` dictionary and ``path``
+        is the relative location of the requested asset on the filesystem.
+        The returned value should be a string, or ``None`` if no version
+        could be determined.
+
+        .. versionchanged:: 3.1
+           This method was previously recommended for subclasses to override;
+           `get_content_version` is now preferred as it allows the base
+           class to handle caching of the result.
+        """
+        abs_path = cls.get_absolute_path(settings['static_path'], path)
+        return cls._get_cached_version(abs_path)
+
+    @classmethod
+    def _get_cached_version(cls, abs_path):
+        with cls._lock:
+            hashes = cls._static_hashes
+            if abs_path not in hashes:
+                try:
+                    hashes[abs_path] = cls.get_content_version(abs_path)
+                except Exception:
+                    gen_log.error("Could not open static file %r", abs_path)
+                    hashes[abs_path] = None
+            hsh = hashes.get(abs_path)
+            if hsh:
+                return hsh
+        return None
+
 
 class FallbackHandler(RequestHandler):
     """A `RequestHandler` that wraps another HTTP server callback.
@@ -2028,6 +2419,22 @@ class TemplateModule(UIModule):
         return "".join(self._get_resources("html_body"))
 
 
+class _UIModuleNamespace(object):
+    """Lazy namespace which creates UIModule proxies bound to a handler."""
+    def __init__(self, handler, ui_modules):
+        self.handler = handler
+        self.ui_modules = ui_modules
+
+    def __getitem__(self, key):
+        return self.handler._ui_module(key, self.ui_modules[key])
+
+    def __getattr__(self, key):
+        try:
+            return self[key]
+        except KeyError as e:
+            raise AttributeError(str(e))
+
+
 class URLSpec(object):
     """Specifies mappings between URLs and handlers."""
     def __init__(self, pattern, handler_class, kwargs=None, name=None):
@@ -2100,7 +2507,7 @@ class URLSpec(object):
         for a in args:
             if not isinstance(a, (unicode_type, bytes_type)):
                 a = str(a)
-            converted_args.append(escape.url_escape(utf8(a)))
+            converted_args.append(escape.url_escape(utf8(a), plus=False))
         return self._path % tuple(converted_args)
 
 url = URLSpec