Update Tornado lib

libs/tornado/platform/auto.py

@@ -32,3 +32,14 @@ if os.name == 'nt':
     from tornado.platform.windows import set_close_exec
 else:
     from tornado.platform.posix import set_close_exec, Waker
+
+try:
+    # monotime monkey-patches the time module to have a monotonic function
+    # in versions of python before 3.3.
+    import monotime
+except ImportError:
+    pass
+try:
+    from time import monotonic as monotonic_time
+except ImportError:
+    monotonic_time = None

libs/tornado/platform/common.py

@@ -69,6 +69,9 @@ class Waker(interface.Waker):
     def fileno(self):
         return self.reader.fileno()
 
+    def write_fileno(self):
+        return self.writer.fileno()
+
     def wake(self):
         try:
             self.writer.send(b("x"))

libs/tornado/platform/epoll.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python
+#
+# Copyright 2012 Facebook
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+"""EPoll-based IOLoop implementation for Linux systems.
+
+Supports the standard library's `select.epoll` function for Python 2.6+,
+and our own C module for Python 2.5.
+"""
+from __future__ import absolute_import, division, with_statement
+
+import os
+import select
+
+from tornado.ioloop import PollIOLoop
+
+if hasattr(select, 'epoll'):
+    # Python 2.6+
+    class EPollIOLoop(PollIOLoop):
+        def initialize(self, **kwargs):
+            super(EPollIOLoop, self).initialize(impl=select.epoll(), **kwargs)
+else:
+    # Python 2.5
+    from tornado import epoll
+    
+    class _EPoll(object):
+        """An epoll-based event loop using our C module for Python 2.5 systems"""
+        _EPOLL_CTL_ADD = 1
+        _EPOLL_CTL_DEL = 2
+        _EPOLL_CTL_MOD = 3
+
+        def __init__(self):
+            self._epoll_fd = epoll.epoll_create()
+
+        def fileno(self):
+            return self._epoll_fd
+
+        def close(self):
+            os.close(self._epoll_fd)
+
+        def register(self, fd, events):
+            epoll.epoll_ctl(self._epoll_fd, self._EPOLL_CTL_ADD, fd, events)
+
+        def modify(self, fd, events):
+            epoll.epoll_ctl(self._epoll_fd, self._EPOLL_CTL_MOD, fd, events)
+
+        def unregister(self, fd):
+            epoll.epoll_ctl(self._epoll_fd, self._EPOLL_CTL_DEL, fd, 0)
+
+        def poll(self, timeout):
+            return epoll.epoll_wait(self._epoll_fd, int(timeout * 1000))
+
+
+    class EPollIOLoop(PollIOLoop):
+        def initialize(self, **kwargs):
+            super(EPollIOLoop, self).initialize(impl=_EPoll(), **kwargs)
+

libs/tornado/platform/interface.py

@@ -39,13 +39,17 @@ class Waker(object):
     the ``IOLoop`` is closed, it closes its waker too.
     """
     def fileno(self):
-        """Returns a file descriptor for this waker.
+        """Returns the read file descriptor for this waker.
 
         Must be suitable for use with ``select()`` or equivalent on the
         local platform.
         """
         raise NotImplementedError()
 
+    def write_fileno(self):
+        """Returns the write file descriptor for this waker."""
+        raise NotImplementedError()
+
     def wake(self):
         """Triggers activity on the waker's file descriptor."""
         raise NotImplementedError()

libs/tornado/platform/kqueue.py

@@ -0,0 +1,91 @@
+#!/usr/bin/env python
+#
+# Copyright 2012 Facebook
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+"""KQueue-based IOLoop implementation for BSD/Mac systems."""
+from __future__ import absolute_import, division, with_statement
+
+import select
+
+from tornado.ioloop import IOLoop, PollIOLoop
+
+assert hasattr(select, 'kqueue'), 'kqueue not supported'
+
+class _KQueue(object):
+    """A kqueue-based event loop for BSD/Mac systems."""
+    def __init__(self):
+        self._kqueue = select.kqueue()
+        self._active = {}
+
+    def fileno(self):
+        return self._kqueue.fileno()
+
+    def close(self):
+        self._kqueue.close()
+
+    def register(self, fd, events):
+        if fd in self._active:
+            raise IOError("fd %d already registered" % fd)
+        self._control(fd, events, select.KQ_EV_ADD)
+        self._active[fd] = events
+
+    def modify(self, fd, events):
+        self.unregister(fd)
+        self.register(fd, events)
+
+    def unregister(self, fd):
+        events = self._active.pop(fd)
+        self._control(fd, events, select.KQ_EV_DELETE)
+
+    def _control(self, fd, events, flags):
+        kevents = []
+        if events & IOLoop.WRITE:
+            kevents.append(select.kevent(
+                    fd, filter=select.KQ_FILTER_WRITE, flags=flags))
+        if events & IOLoop.READ or not kevents:
+            # Always read when there is not a write
+            kevents.append(select.kevent(
+                    fd, filter=select.KQ_FILTER_READ, flags=flags))
+        # Even though control() takes a list, it seems to return EINVAL
+        # on Mac OS X (10.6) when there is more than one event in the list.
+        for kevent in kevents:
+            self._kqueue.control([kevent], 0)
+
+    def poll(self, timeout):
+        kevents = self._kqueue.control(None, 1000, timeout)
+        events = {}
+        for kevent in kevents:
+            fd = kevent.ident
+            if kevent.filter == select.KQ_FILTER_READ:
+                events[fd] = events.get(fd, 0) | IOLoop.READ
+            if kevent.filter == select.KQ_FILTER_WRITE:
+                if kevent.flags & select.KQ_EV_EOF:
+                    # If an asynchronous connection is refused, kqueue
+                    # returns a write event with the EOF flag set.
+                    # Turn this into an error for consistency with the
+                    # other IOLoop implementations.
+                    # Note that for read events, EOF may be returned before
+                    # all data has been consumed from the socket buffer,
+                    # so we only check for EOF on write events.
+                    events[fd] = IOLoop.ERROR
+                else:
+                    events[fd] = events.get(fd, 0) | IOLoop.WRITE
+            if kevent.flags & select.KQ_EV_ERROR:
+                events[fd] = events.get(fd, 0) | IOLoop.ERROR
+        return events.items()
+
+
+class KQueueIOLoop(PollIOLoop):
+    def initialize(self, **kwargs):
+        super(KQueueIOLoop, self).initialize(impl=_KQueue(), **kwargs)

libs/tornado/platform/posix.py

@@ -48,6 +48,9 @@ class Waker(interface.Waker):
     def fileno(self):
         return self.reader.fileno()
 
+    def write_fileno(self):
+        return self.writer.fileno()
+
     def wake(self):
         try:
             self.writer.write(b("x"))

libs/tornado/platform/select.py

@@ -0,0 +1,75 @@
+#!/usr/bin/env python
+#
+# Copyright 2012 Facebook
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+"""Select-based IOLoop implementation.
+
+Used as a fallback for systems that don't support epoll or kqueue.
+"""
+from __future__ import absolute_import, division, with_statement
+
+import select
+
+from tornado.ioloop import IOLoop, PollIOLoop
+
+class _Select(object):
+    """A simple, select()-based IOLoop implementation for non-Linux systems"""
+    def __init__(self):
+        self.read_fds = set()
+        self.write_fds = set()
+        self.error_fds = set()
+        self.fd_sets = (self.read_fds, self.write_fds, self.error_fds)
+
+    def close(self):
+        pass
+
+    def register(self, fd, events):
+        if fd in self.read_fds or fd in self.write_fds or fd in self.error_fds:
+            raise IOError("fd %d already registered" % fd)
+        if events & IOLoop.READ:
+            self.read_fds.add(fd)
+        if events & IOLoop.WRITE:
+            self.write_fds.add(fd)
+        if events & IOLoop.ERROR:
+            self.error_fds.add(fd)
+            # Closed connections are reported as errors by epoll and kqueue,
+            # but as zero-byte reads by select, so when errors are requested
+            # we need to listen for both read and error.
+            self.read_fds.add(fd)
+
+    def modify(self, fd, events):
+        self.unregister(fd)
+        self.register(fd, events)
+
+    def unregister(self, fd):
+        self.read_fds.discard(fd)
+        self.write_fds.discard(fd)
+        self.error_fds.discard(fd)
+
+    def poll(self, timeout):
+        readable, writeable, errors = select.select(
+            self.read_fds, self.write_fds, self.error_fds, timeout)
+        events = {}
+        for fd in readable:
+            events[fd] = events.get(fd, 0) | IOLoop.READ
+        for fd in writeable:
+            events[fd] = events.get(fd, 0) | IOLoop.WRITE
+        for fd in errors:
+            events[fd] = events.get(fd, 0) | IOLoop.ERROR
+        return events.items()
+
+class SelectIOLoop(PollIOLoop):
+    def initialize(self, **kwargs):
+        super(SelectIOLoop, self).initialize(impl=_Select(), **kwargs)
+

libs/tornado/platform/twisted.py

@@ -16,19 +16,25 @@
 # Note:  This module's docs are not currently extracted automatically,
 # so changes must be made manually to twisted.rst
 # TODO: refactor doc build process to use an appropriate virtualenv
-"""A Twisted reactor built on the Tornado IOLoop.
+"""Bridges between the Twisted reactor and Tornado IOLoop.
 
 This module lets you run applications and libraries written for
-Twisted in a Tornado application.  To use it, simply call `install` at
-the beginning of the application::
+Twisted in a Tornado application.  It can be used in two modes,
+depending on which library's underlying event loop you want to use.
+
+Twisted on Tornado
+------------------
+
+`TornadoReactor` implements the Twisted reactor interface on top of
+the Tornado IOLoop.  To use it, simply call `install` at the beginning
+of the application::
 
     import tornado.platform.twisted
     tornado.platform.twisted.install()
     from twisted.internet import reactor
 
 When the app is ready to start, call `IOLoop.instance().start()`
-instead of `reactor.run()`.  This will allow you to use a mixture of
-Twisted and Tornado code in the same process.
+instead of `reactor.run()`.
 
 It is also possible to create a non-global reactor by calling
 `tornado.platform.twisted.TornadoReactor(io_loop)`.  However, if
@@ -41,18 +47,32 @@ recommended to call::
 
 before closing the `IOLoop`.
 
-This module has been tested with Twisted versions 11.0.0, 11.1.0, and 12.0.0
+Tornado on Twisted
+------------------
+
+`TwistedIOLoop` implements the Tornado IOLoop interface on top of the Twisted
+reactor.  Recommended usage::
+
+    from tornado.platform.twisted import TwistedIOLoop
+    from twisted.internet import reactor
+    TwistedIOLoop().install()
+    # Set up your tornado application as usual using `IOLoop.instance`
+    reactor.run()
+
+`TwistedIOLoop` always uses the global Twisted reactor.
+
+This module has been tested with Twisted versions 11.0.0 and newer.
 """
 
 from __future__ import absolute_import, division, with_statement
 
 import functools
-import logging
+import datetime
 import time
 
 from twisted.internet.posixbase import PosixReactorBase
 from twisted.internet.interfaces import \
-    IReactorFDSet, IDelayedCall, IReactorTime
+    IReactorFDSet, IDelayedCall, IReactorTime, IReadDescriptor, IWriteDescriptor
 from twisted.python import failure, log
 from twisted.internet import error
 
@@ -60,7 +80,8 @@ from zope.interface import implementer
 
 import tornado
 import tornado.ioloop
-from tornado.stack_context import NullContext
+from tornado.log import app_log
+from tornado.stack_context import NullContext, wrap
 from tornado.ioloop import IOLoop
 
 
@@ -80,7 +101,7 @@ class TornadoDelayedCall(object):
         try:
             self._func()
         except:
-            logging.error("_called caught exception", exc_info=True)
+            app_log.error("_called caught exception", exc_info=True)
 
     def getTime(self):
         return self._time
@@ -127,6 +148,7 @@ class TornadoReactor(PosixReactorBase):
         self._fds = {}  # a map of fd to a (reader, writer) tuple
         self._delayedCalls = {}
         PosixReactorBase.__init__(self)
+        self.addSystemEventTrigger('during', 'shutdown', self.crash)
 
         # IOLoop.start() bypasses some of the reactor initialization.
         # Fire off the necessary events if they weren't already triggered
@@ -138,7 +160,7 @@ class TornadoReactor(PosixReactorBase):
 
     # IReactorTime
     def seconds(self):
-        return time.time()
+        return self._io_loop.time()
 
     def callLater(self, seconds, f, *args, **kw):
         dc = TornadoDelayedCall(self, seconds, f, *args, **kw)
@@ -169,6 +191,8 @@ class TornadoReactor(PosixReactorBase):
 
     # IReactorFDSet
     def _invoke_callback(self, fd, events):
+        if fd not in self._fds:
+            return
         (reader, writer) = self._fds[fd]
         if reader:
             err = None
@@ -280,7 +304,8 @@ class TornadoReactor(PosixReactorBase):
     # IOLoop.start() instead of Reactor.run().
     def stop(self):
         PosixReactorBase.stop(self)
-        self._io_loop.stop()
+        fire_shutdown = functools.partial(self.fireSystemEvent, "shutdown")
+        self._io_loop.add_callback(fire_shutdown)
 
     def crash(self):
         PosixReactorBase.crash(self)
@@ -291,8 +316,6 @@ class TornadoReactor(PosixReactorBase):
 
     def mainLoop(self):
         self._io_loop.start()
-        if self._stopped:
-            self.fireSystemEvent("shutdown")
 TornadoReactor = implementer(IReactorTime, IReactorFDSet)(TornadoReactor)
 
 
@@ -328,3 +351,113 @@ def install(io_loop=None):
     from twisted.internet.main import installReactor
     installReactor(reactor)
     return reactor
+
+class _FD(object):
+    def __init__(self, fd, handler):
+        self.fd = fd
+        self.handler = handler
+        self.reading = False
+        self.writing = False
+        self.lost = False
+
+    def fileno(self):
+        return self.fd
+
+    def doRead(self):
+        if not self.lost:
+            self.handler(self.fd, tornado.ioloop.IOLoop.READ)
+
+    def doWrite(self):
+        if not self.lost:
+            self.handler(self.fd, tornado.ioloop.IOLoop.WRITE)
+
+    def connectionLost(self, reason):
+        if not self.lost:
+            self.handler(self.fd, tornado.ioloop.IOLoop.ERROR)
+            self.lost = True
+
+    def logPrefix(self):
+        return ''
+_FD = implementer(IReadDescriptor, IWriteDescriptor)(_FD)
+
+class TwistedIOLoop(tornado.ioloop.IOLoop):
+    """IOLoop implementation that runs on Twisted.
+
+    Uses the global Twisted reactor.  It is possible to create multiple
+    TwistedIOLoops in the same process, but it doesn't really make sense
+    because they will all run in the same thread.
+
+    Not compatible with `tornado.process.Subprocess.set_exit_callback`
+    because the ``SIGCHLD`` handlers used by Tornado and Twisted conflict
+    with each other.
+    """
+    def initialize(self):
+        from twisted.internet import reactor
+        self.reactor = reactor
+        self.fds = {}
+
+    def close(self, all_fds=False):
+        self.reactor.removeAll()
+        for c in self.reactor.getDelayedCalls():
+            c.cancel()
+
+    def add_handler(self, fd, handler, events):
+        if fd in self.fds:
+            raise ValueError('fd %d added twice' % fd)
+        self.fds[fd] = _FD(fd, wrap(handler))
+        if events | tornado.ioloop.IOLoop.READ:
+            self.fds[fd].reading = True
+            self.reactor.addReader(self.fds[fd])
+        if events | tornado.ioloop.IOLoop.WRITE:
+            self.fds[fd].writing = True
+            self.reactor.addWriter(self.fds[fd])
+
+    def update_handler(self, fd, events):
+        if events | tornado.ioloop.IOLoop.READ:
+            if not self.fds[fd].reading:
+                self.fds[fd].reading = True
+                self.reactor.addReader(self.fds[fd])
+        else:
+            if self.fds[fd].reading:
+                self.fds[fd].reading = False
+                self.reactor.removeReader(self.fds[fd])
+        if events | tornado.ioloop.IOLoop.WRITE:
+            if not self.fds[fd].writing:
+                self.fds[fd].writing = True
+                self.reactor.addWriter(self.fds[fd])
+        else:
+            if self.fds[fd].writing:
+                self.fds[fd].writing = False
+                self.reactor.removeWriter(self.fds[fd])
+
+    def remove_handler(self, fd):
+        self.fds[fd].lost = True
+        if self.fds[fd].reading:
+            self.reactor.removeReader(self.fds[fd])
+        if self.fds[fd].writing:
+            self.reactor.removeWriter(self.fds[fd])
+        del self.fds[fd]
+
+    def start(self):
+        self.reactor.run()
+
+    def stop(self):
+        self.reactor.crash()
+
+    def add_timeout(self, deadline, callback):
+        if isinstance(deadline, (int, long, float)):
+            delay = max(deadline - self.time(), 0)
+        elif isinstance(deadline, datetime.timedelta):
+            delay = deadline.total_seconds()
+        else:
+            raise TypeError("Unsupported deadline %r")
+        return self.reactor.callLater(delay, wrap(callback))
+
+    def remove_timeout(self, timeout):
+        timeout.cancel()
+
+    def add_callback(self, callback):
+        self.reactor.callFromThread(wrap(callback))
+
+    def add_callback_from_signal(self, callback):
+        self.add_callback(callback)