Update to Tornado 3.0

libs/tornado/gen.py

@@ -20,29 +20,41 @@ could be written with ``gen`` as::
 
     class GenAsyncHandler(RequestHandler):
         @asynchronous
-        @gen.engine
+        @gen.coroutine
         def get(self):
             http_client = AsyncHTTPClient()
-            response = yield gen.Task(http_client.fetch, "http://example.com")
+            response = yield http_client.fetch("http://example.com")
             do_something_with_response(response)
             self.render("template.html")
 
-`Task` works with any function that takes a ``callback`` keyword
-argument.  You can also yield a list of ``Tasks``, which will be
+Most asynchronous functions in Tornado return a `.Future`;
+yielding this object returns its `~.Future.result`.
+
+For functions that do not return ``Futures``, `Task` works with any
+function that takes a ``callback`` keyword argument (most Tornado functions
+can be used in either style, although the ``Future`` style is preferred
+since it is both shorter and provides better exception handling)::
+
+    @gen.coroutine
+    def get(self):
+        yield gen.Task(AsyncHTTPClient().fetch, "http://example.com")
+
+You can also yield a list of ``Futures`` and/or ``Tasks``, which will be
 started at the same time and run in parallel; a list of results will
 be returned when they are all finished::
 
+    @gen.coroutine
     def get(self):
         http_client = AsyncHTTPClient()
-        response1, response2 = yield [gen.Task(http_client.fetch, url1),
-                                      gen.Task(http_client.fetch, url2)]
+        response1, response2 = yield [http_client.fetch(url1),
+                                      http_client.fetch(url2)]
 
 For more complicated interfaces, `Task` can be split into two parts:
 `Callback` and `Wait`::
 
     class GenAsyncHandler2(RequestHandler):
         @asynchronous
-        @gen.engine
+        @gen.coroutine
         def get(self):
             http_client = AsyncHTTPClient()
             http_client.fetch("http://example.com",
@@ -70,9 +82,9 @@ import itertools
 import sys
 import types
 
-from tornado.concurrent import Future
+from tornado.concurrent import Future, TracebackFuture
 from tornado.ioloop import IOLoop
-from tornado.stack_context import ExceptionStackContext
+from tornado.stack_context import ExceptionStackContext, wrap
 
 
 class KeyReuseError(Exception):
@@ -91,18 +103,26 @@ class BadYieldError(Exception):
     pass
 
 
-def engine(func):
-    """Decorator for asynchronous generators.
+class ReturnValueIgnoredError(Exception):
+    pass
 
-    Any generator that yields objects from this module must be wrapped
-    in this decorator.  The decorator only works on functions that are
-    already asynchronous.  For `~tornado.web.RequestHandler`
-    ``get``/``post``/etc methods, this means that both the
-    `tornado.web.asynchronous` and `tornado.gen.engine` decorators
-    must be used (for proper exception handling, ``asynchronous``
-    should come before ``gen.engine``).  In most other cases, it means
-    that it doesn't make sense to use ``gen.engine`` on functions that
-    don't already take a callback argument.
+
+def engine(func):
+    """Callback-oriented decorator for asynchronous generators.
+
+    This is an older interface; for new code that does not need to be
+    compatible with versions of Tornado older than 3.0 the
+    `coroutine` decorator is recommended instead.
+
+    This decorator is similar to `coroutine`, except it does not
+    return a `.Future` and the ``callback`` argument is not treated
+    specially.
+
+    In most cases, functions decorated with `engine` should take
+    a ``callback`` argument and invoke it with their result when
+    they are finished.  One notable exception is the
+    `~tornado.web.RequestHandler` :ref:`HTTP verb methods <verbs>`,
+    which use ``self.finish()`` in place of a callback argument.
     """
     @functools.wraps(func)
     def wrapper(*args, **kwargs):
@@ -116,20 +136,129 @@ def engine(func):
             if runner is not None:
                 return runner.handle_exception(typ, value, tb)
             return False
-        with ExceptionStackContext(handle_exception) as deactivate:
-            gen = func(*args, **kwargs)
-            if isinstance(gen, types.GeneratorType):
-                runner = Runner(gen, deactivate)
-                runner.run()
-                return
-            assert gen is None, gen
-            deactivate()
+        with ExceptionStackContext(handle_exception):
+            try:
+                result = func(*args, **kwargs)
+            except (Return, StopIteration) as e:
+                result = getattr(e, 'value', None)
+            else:
+                if isinstance(result, types.GeneratorType):
+                    def final_callback(value):
+                        if value is not None:
+                            raise ReturnValueIgnoredError(
+                                "@gen.engine functions cannot return values: "
+                                "%r" % (value,))
+                        assert value is None
+                    runner = Runner(result, final_callback)
+                    runner.run()
+                    return
+            if result is not None:
+                raise ReturnValueIgnoredError(
+                    "@gen.engine functions cannot return values: %r" %
+                    (result,))
             # no yield, so we're done
     return wrapper
 
 
+def coroutine(func):
+    """Decorator for asynchronous generators.
+
+    Any generator that yields objects from this module must be wrapped
+    in either this decorator or `engine`.  These decorators only work
+    on functions that are already asynchronous.  For
+    `~tornado.web.RequestHandler` :ref:`HTTP verb methods <verbs>` methods, this
+    means that both the `tornado.web.asynchronous` and
+    `tornado.gen.coroutine` decorators must be used (for proper
+    exception handling, ``asynchronous`` should come before
+    ``gen.coroutine``).
+
+    Coroutines may "return" by raising the special exception
+    `Return(value) <Return>`.  In Python 3.3+, it is also possible for
+    the function to simply use the ``return value`` statement (prior to
+    Python 3.3 generators were not allowed to also return values).
+    In all versions of Python a coroutine that simply wishes to exit
+    early may use the ``return`` statement without a value.
+
+    Functions with this decorator return a `.Future`.  Additionally,
+    they may be called with a ``callback`` keyword argument, which
+    will be invoked with the future's result when it resolves.  If the
+    coroutine fails, the callback will not be run and an exception
+    will be raised into the surrounding `.StackContext`.  The
+    ``callback`` argument is not visible inside the decorated
+    function; it is handled by the decorator itself.
+
+    From the caller's perspective, ``@gen.coroutine`` is similar to
+    the combination of ``@return_future`` and ``@gen.engine``.
+    """
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        runner = None
+        future = TracebackFuture()
+
+        if 'callback' in kwargs:
+            callback = kwargs.pop('callback')
+            IOLoop.current().add_future(
+                future, lambda future: callback(future.result()))
+
+        def handle_exception(typ, value, tb):
+            try:
+                if runner is not None and runner.handle_exception(typ, value, tb):
+                    return True
+            except Exception:
+                typ, value, tb = sys.exc_info()
+            future.set_exc_info((typ, value, tb))
+            return True
+        with ExceptionStackContext(handle_exception):
+            try:
+                result = func(*args, **kwargs)
+            except (Return, StopIteration) as e:
+                result = getattr(e, 'value', None)
+            except Exception:
+                future.set_exc_info(sys.exc_info())
+                return future
+            else:
+                if isinstance(result, types.GeneratorType):
+                    def final_callback(value):
+                        future.set_result(value)
+                    runner = Runner(result, final_callback)
+                    runner.run()
+                    return future
+            future.set_result(result)
+        return future
+    return wrapper
+
+
+class Return(Exception):
+    """Special exception to return a value from a `coroutine`.
+
+    If this exception is raised, its value argument is used as the
+    result of the coroutine::
+
+        @gen.coroutine
+        def fetch_json(url):
+            response = yield AsyncHTTPClient().fetch(url)
+            raise gen.Return(json_decode(response.body))
+
+    In Python 3.3, this exception is no longer necessary: the ``return``
+    statement can be used directly to return a value (previously
+    ``yield`` and ``return`` with a value could not be combined in the
+    same function).
+
+    By analogy with the return statement, the value argument is optional,
+    but it is never necessary to ``raise gen.Return()``.  The ``return``
+    statement can be used with no arguments instead.
+    """
+    def __init__(self, value=None):
+        super(Return, self).__init__()
+        self.value = value
+
+
 class YieldPoint(object):
-    """Base class for objects that may be yielded from the generator."""
+    """Base class for objects that may be yielded from the generator.
+
+    Applications do not normally need to use this class, but it may be
+    subclassed to provide additional yielding behavior.
+    """
     def start(self, runner):
         """Called by the runner after the generator has yielded.
 
@@ -195,7 +324,7 @@ class Wait(YieldPoint):
 
 
 class WaitAll(YieldPoint):
-    """Returns the results of multiple previous `Callbacks`.
+    """Returns the results of multiple previous `Callbacks <Callback>`.
 
     The argument is a sequence of `Callback` keys, and the result is
     a list of results in the same order.
@@ -291,7 +420,7 @@ class Multi(YieldPoint):
 
     def is_ready(self):
         finished = list(itertools.takewhile(
-                lambda i: i.is_ready(), self.unfinished_children))
+            lambda i: i.is_ready(), self.unfinished_children))
         self.unfinished_children.difference_update(finished)
         return not self.unfinished_children
 
@@ -331,13 +460,13 @@ class Runner(object):
     def register_callback(self, key):
         """Adds ``key`` to the list of callbacks."""
         if key in self.pending_callbacks:
-            raise KeyReuseError("key %r is already pending" % key)
+            raise KeyReuseError("key %r is already pending" % (key,))
         self.pending_callbacks.add(key)
 
     def is_ready(self, key):
         """Returns true if a result is available for ``key``."""
         if key not in self.pending_callbacks:
-            raise UnknownKeyError("key %r is not pending" % key)
+            raise UnknownKeyError("key %r is not pending" % (key,))
         return key in self.results
 
     def set_result(self, key, result):
@@ -374,7 +503,7 @@ class Runner(object):
                         yielded = self.gen.throw(*exc_info)
                     else:
                         yielded = self.gen.send(next)
-                except StopIteration:
+                except (StopIteration, Return) as e:
                     self.finished = True
                     if self.pending_callbacks and not self.had_exception:
                         # If we ran cleanly without waiting on all callbacks
@@ -384,7 +513,7 @@ class Runner(object):
                         raise LeakedCallbackError(
                             "finished without waiting for callbacks %r" %
                             self.pending_callbacks)
-                    self.final_callback()
+                    self.final_callback(getattr(e, 'value', None))
                     self.final_callback = None
                     return
                 except Exception:
@@ -401,7 +530,8 @@ class Runner(object):
                     except Exception:
                         self.exc_info = sys.exc_info()
                 else:
-                    self.exc_info = (BadYieldError("yielded unknown object %r" % yielded),)
+                    self.exc_info = (BadYieldError(
+                            "yielded unknown object %r" % (yielded,)),)
         finally:
             self.running = False
 
@@ -414,7 +544,7 @@ class Runner(object):
             else:
                 result = None
             self.set_result(key, result)
-        return inner
+        return wrap(inner)
 
     def handle_exception(self, typ, value, tb):
         if not self.running and not self.finished: