Merge branch 'refs/heads/develop'

couchpotato/core/media/_base/providers/nzb/newznab.py

@@ -1,9 +1,7 @@
-from urllib2 import HTTPError
 from urlparse import urlparse
 import time
 import traceback
 import re
-import urllib2
 
 from couchpotato.core.helpers.encoding import tryUrlencode, toUnicode
 from couchpotato.core.helpers.rss import RSS
@@ -13,6 +11,7 @@ from couchpotato.core.media._base.providers.base import ResultList
 from couchpotato.core.media._base.providers.nzb.base import NZBProvider
 from couchpotato.environment import Env
 from dateutil.parser import parse
+from requests import HTTPError
 
 
 log = CPLog(__name__)
@@ -184,16 +183,7 @@ class Base(NZBProvider, RSS):
                 return 'try_next'
 
         try:
-            # Get final redirected url
+            data = self.urlopen(url, show_error = False)
-            log.debug('Checking %s for redirects.', url)
-            req = urllib2.Request(url)
-            req.add_header('User-Agent', self.user_agent)
-            res = urllib2.urlopen(req)
-            finalurl = res.geturl()
-            if finalurl != url:
-                log.debug('Redirect url used: %s', finalurl)
-
-            data = self.urlopen(finalurl, show_error = False)
             self.limits_reached[host] = False
             return data
         except HTTPError as e:

couchpotato/core/media/movie/providers/info/fanarttv.py

@@ -74,7 +74,7 @@ class FanartTV(MovieProvider):
         fanarts = self._getMultImages(movie.get('moviebackground', []), self.MAX_EXTRAFANART + 1)
 
         if fanarts:
-            images['backdrop_original'] = fanarts[0]
+            images['backdrop_original'] = [fanarts[0]]
             images['extra_fanart'] = fanarts[1:]
 
         return images

couchpotato/core/media/movie/providers/metadata/wmc.py

@@ -8,7 +8,7 @@ autoload = 'WindowsMediaCenter'
 
 class WindowsMediaCenter(MovieMetaData):
 
-    def getThumbnailName(self, name, root):
+    def getThumbnailName(self, name, root, i):
         return os.path.join(root, 'folder.jpg')
 
 

couchpotato/core/media/movie/providers/metadata/xbmc.py

@@ -303,7 +303,7 @@ config = [{
                     'advanced': True,
                 },
                 {
-                    'name': 'meta_disc',
+                    'name': 'meta_disc_art',
                     'label': 'DiscArt',
                     'default': False,
                     'type': 'bool'

couchpotato/core/media/movie/providers/trailer/hdtrailers.py

@@ -1,5 +1,4 @@
 from string import digits, ascii_letters
-from urllib2 import HTTPError
 import re
 
 from bs4 import SoupStrainer, BeautifulSoup
@@ -7,6 +6,7 @@ from couchpotato.core.helpers.encoding import tryUrlencode
 from couchpotato.core.helpers.variable import mergeDicts, getTitle
 from couchpotato.core.logger import CPLog
 from couchpotato.core.media.movie.providers.trailer.base import TrailerProvider
+from requests import HTTPError
 
 
 log = CPLog(__name__)

couchpotato/core/plugins/base.py

@@ -1,3 +1,4 @@
+from urllib import quote
 from urlparse import urlparse
 import glob
 import inspect
@@ -5,7 +6,6 @@ import os.path
 import re
 import time
 import traceback
-import urllib2
 
 from couchpotato.core.event import fireEvent, addEvent
 from couchpotato.core.helpers.encoding import ss, toSafeString, \
@@ -167,7 +167,7 @@ class Plugin(object):
 
     # http request
     def urlopen(self, url, timeout = 30, data = None, headers = None, files = None, show_error = True, verify_ssl = True):
-        url = urllib2.quote(ss(url), safe = "%/:=&?~#+!$,;'@()*[]")
+        url = quote(ss(url), safe = "%/:=&?~#+!$,;'@()*[]")
 
         if not headers: headers = {}
         if not data: data = {}

couchpotato/core/plugins/manage.py

@@ -4,7 +4,6 @@ import sys
 import time
 import traceback
 
-from couchpotato import get_db
 from couchpotato.api import addApiView
 from couchpotato.core.event import fireEvent, addEvent, fireEventAsync
 from couchpotato.core.helpers.encoding import sp
@@ -71,7 +70,8 @@ class Manage(Plugin):
         return self.updateLibrary(full = False)
 
     def updateLibrary(self, full = True):
-        last_update = float(Env.prop('manage.last_update', default = 0))
+        last_update_key = 'manage.last_update%s' % ('_full' if full else '')
+        last_update = float(Env.prop(last_update_key, default = 0))
 
         if self.in_progress:
             log.info('Already updating library: %s', self.in_progress)
@@ -162,7 +162,7 @@ class Manage(Plugin):
                                             used_files[release_file] = release
                             del used_files
 
-            Env.prop('manage.last_update', time.time())
+            Env.prop(last_update_key, time.time())
         except:
             log.error('Failed updating library: %s', (traceback.format_exc()))
 

couchpotato/core/plugins/quality/main.py

@@ -37,7 +37,7 @@ class QualityPlugin(Plugin):
     threed_tags = {
         'sbs': [('half', 'sbs'), 'hsbs', ('full', 'sbs'), 'fsbs'],
         'ou': [('half', 'ou'), 'hou', ('full', 'ou'), 'fou'],
-        '3d': ['2d3d', '3d2d'],
+        '3d': ['2d3d', '3d2d', '3d'],
     }
 
     cached_qualities = None
@@ -290,14 +290,14 @@ class QualityPlugin(Plugin):
             tags = self.threed_tags.get(key, [])
 
             for tag in tags:
-                if (isinstance(tag, tuple) and '.'.join(tag) in '.'.join(words)) or (isinstance(tag, (str, unicode)) and ss(tag.lower()) in cur_file.lower()):
+                if isinstance(tag, tuple):
+                    if len(set(words) & set(tag)) == len(tag):
+                        log.debug('Found %s in %s', (tag, cur_file))
+                        return 1, key
+                elif tag in words:
                     log.debug('Found %s in %s', (tag, cur_file))
                     return 1, key
 
-            if list(set([key]) & set(words)):
-                log.debug('Found %s in %s', (key, cur_file))
-                return 1, key
-
         return 0, None
 
     def guessLooseScore(self, quality, extra = None):
@@ -423,6 +423,8 @@ class QualityPlugin(Plugin):
             'Movie Monuments 2013 BrRip 1080p': {'size': 1800, 'quality': 'brrip'},
             'Movie Monuments 2013 BrRip 720p': {'size': 1300, 'quality': 'brrip'},
             'The.Movie.2014.3D.1080p.BluRay.AVC.DTS-HD.MA.5.1-GroupName': {'size': 30000, 'quality': 'bd50', 'is_3d': True},
+            '/home/namehou/Movie Monuments (2013)/Movie Monuments.mkv': {'size': 4500, 'quality': '1080p', 'is_3d': False},
+            '/home/namehou/Movie Monuments (2013)/Movie Monuments Full-OU.mkv': {'size': 4500, 'quality': '1080p', 'is_3d': True}
         }
 
         correct = 0

couchpotato/core/plugins/release/main.py

@@ -167,6 +167,9 @@ class Release(Plugin):
             rel = db.get('id', release_id)
             db.delete(rel)
             return True
+        except RecordDeleted:
+            log.error('Already deleted: %s', release_id)
+            return True
         except:
             log.error('Failed: %s', traceback.format_exc())
 

couchpotato/core/settings.py

@@ -49,7 +49,7 @@ class Settings(object):
             'desc': 'Save setting to config file (settings.conf)',
             'params': {
                 'section': {'desc': 'The section name in settings.conf'},
-                'option': {'desc': 'The option name'},
+                'name': {'desc': 'The option name'},
                 'value': {'desc': 'The value you want to save'},
             }
         })

libs/tornado/auth.py

@@ -34,15 +34,29 @@ See the individual service classes below for complete documentation.
 
 Example usage for Google OpenID::
 
-    class GoogleLoginHandler(tornado.web.RequestHandler,
+    class GoogleOAuth2LoginHandler(tornado.web.RequestHandler,
-                             tornado.auth.GoogleMixin):
+                                   tornado.auth.GoogleOAuth2Mixin):
         @tornado.gen.coroutine
         def get(self):
-            if self.get_argument("openid.mode", None):
+            if self.get_argument('code', False):
-                user = yield self.get_authenticated_user()
+                user = yield self.get_authenticated_user(
-                # Save the user with e.g. set_secure_cookie()
+                    redirect_uri='http://your.site.com/auth/google',
+                    code=self.get_argument('code'))
+                # Save the user with e.g. set_secure_cookie
             else:
-                yield self.authenticate_redirect()
+                yield self.authorize_redirect(
+                    redirect_uri='http://your.site.com/auth/google',
+                    client_id=self.settings['google_oauth']['key'],
+                    scope=['profile', 'email'],
+                    response_type='code',
+                    extra_params={'approval_prompt': 'auto'})
+
+.. versionchanged:: 3.3
+   All of the callback interfaces in this module are now guaranteed
+   to run their callback with an argument of ``None`` on error.
+   Previously some functions would do this while others would simply
+   terminate the request on their own.  This change also ensures that
+   errors are more consistently reported through the ``Future`` interfaces.
 """
 
 from __future__ import absolute_import, division, print_function, with_statement
@@ -61,6 +75,7 @@ from tornado import httpclient
 from tornado import escape
 from tornado.httputil import url_concat
 from tornado.log import gen_log
+from tornado.stack_context import ExceptionStackContext
 from tornado.util import bytes_type, u, unicode_type, ArgReplacer
 
 try:
@@ -108,7 +123,14 @@ def _auth_return_future(f):
         if callback is not None:
             future.add_done_callback(
                 functools.partial(_auth_future_to_callback, callback))
-        f(*args, **kwargs)
+        def handle_exception(typ, value, tb):
+            if future.done():
+                return False
+            else:
+                future.set_exc_info((typ, value, tb))
+                return True
+        with ExceptionStackContext(handle_exception):
+            f(*args, **kwargs)
         return future
     return wrapper
 
@@ -166,7 +188,7 @@ class OpenIdMixin(object):
         url = self._OPENID_ENDPOINT
         if http_client is None:
             http_client = self.get_auth_http_client()
-        http_client.fetch(url, self.async_callback(
+        http_client.fetch(url, functools.partial(
             self._on_authentication_verified, callback),
             method="POST", body=urllib_parse.urlencode(args))
 
@@ -338,7 +360,7 @@ class OAuthMixin(object):
             http_client.fetch(
                 self._oauth_request_token_url(callback_uri=callback_uri,
                                               extra_params=extra_params),
-                self.async_callback(
+                functools.partial(
                     self._on_request_token,
                     self._OAUTH_AUTHORIZE_URL,
                     callback_uri,
@@ -346,7 +368,7 @@ class OAuthMixin(object):
         else:
             http_client.fetch(
                 self._oauth_request_token_url(),
-                self.async_callback(
+                functools.partial(
                     self._on_request_token, self._OAUTH_AUTHORIZE_URL,
                     callback_uri,
                     callback))
@@ -383,7 +405,7 @@ class OAuthMixin(object):
         if http_client is None:
             http_client = self.get_auth_http_client()
         http_client.fetch(self._oauth_access_token_url(token),
-                          self.async_callback(self._on_access_token, callback))
+                          functools.partial(self._on_access_token, callback))
 
     def _oauth_request_token_url(self, callback_uri=None, extra_params=None):
         consumer_token = self._oauth_consumer_token()
@@ -460,7 +482,7 @@ class OAuthMixin(object):
 
         access_token = _oauth_parse_response(response.body)
         self._oauth_get_user_future(access_token).add_done_callback(
-            self.async_callback(self._on_oauth_get_user, access_token, future))
+            functools.partial(self._on_oauth_get_user, access_token, future))
 
     def _oauth_consumer_token(self):
         """Subclasses must override this to return their OAuth consumer keys.
@@ -645,7 +667,7 @@ class TwitterMixin(OAuthMixin):
         """
         http = self.get_auth_http_client()
         http.fetch(self._oauth_request_token_url(callback_uri=callback_uri),
-                   self.async_callback(
+                   functools.partial(
                        self._on_request_token, self._OAUTH_AUTHENTICATE_URL,
                        None, callback))
 
@@ -703,7 +725,7 @@ class TwitterMixin(OAuthMixin):
         if args:
             url += "?" + urllib_parse.urlencode(args)
         http = self.get_auth_http_client()
-        http_callback = self.async_callback(self._on_twitter_request, callback)
+        http_callback = functools.partial(self._on_twitter_request, callback)
         if post_args is not None:
             http.fetch(url, method="POST", body=urllib_parse.urlencode(post_args),
                        callback=http_callback)
@@ -820,7 +842,7 @@ class FriendFeedMixin(OAuthMixin):
             args.update(oauth)
         if args:
             url += "?" + urllib_parse.urlencode(args)
-        callback = self.async_callback(self._on_friendfeed_request, callback)
+        callback = functools.partial(self._on_friendfeed_request, callback)
         http = self.get_auth_http_client()
         if post_args is not None:
             http.fetch(url, method="POST", body=urllib_parse.urlencode(post_args),
@@ -861,6 +883,10 @@ class FriendFeedMixin(OAuthMixin):
 class GoogleMixin(OpenIdMixin, OAuthMixin):
     """Google Open ID / OAuth authentication.
 
+    *Deprecated:* New applications should use `GoogleOAuth2Mixin`
+    below instead of this class. As of May 19, 2014, Google has stopped
+    supporting registration-free authentication.
+
     No application registration is necessary to use Google for
     authentication or to access Google resources on behalf of a user.
 
@@ -931,7 +957,7 @@ class GoogleMixin(OpenIdMixin, OAuthMixin):
             http = self.get_auth_http_client()
             token = dict(key=token, secret="")
             http.fetch(self._oauth_access_token_url(token),
-                       self.async_callback(self._on_access_token, callback))
+                       functools.partial(self._on_access_token, callback))
         else:
             chain_future(OpenIdMixin.get_authenticated_user(self),
                          callback)
@@ -950,6 +976,19 @@ class GoogleMixin(OpenIdMixin, OAuthMixin):
 class GoogleOAuth2Mixin(OAuth2Mixin):
     """Google authentication using OAuth2.
 
+    In order to use, register your application with Google and copy the
+    relevant parameters to your application settings.
+
+    * Go to the Google Dev Console at http://console.developers.google.com
+    * Select a project, or create a new one.
+    * In the sidebar on the left, select APIs & Auth.
+    * In the list of APIs, find the Google+ API service and set it to ON.
+    * In the sidebar on the left, select Credentials.
+    * In the OAuth section of the page, select Create New Client ID.
+    * Set the Redirect URI to point to your auth handler
+    * Copy the "Client secret" and "Client ID" to the application settings as
+      {"google_oauth": {"key": CLIENT_ID, "secret": CLIENT_SECRET}}
+
     .. versionadded:: 3.2
     """
     _OAUTH_AUTHORIZE_URL = "https://accounts.google.com/o/oauth2/auth"
@@ -963,7 +1002,7 @@ class GoogleOAuth2Mixin(OAuth2Mixin):
 
         Example usage::
 
-            class GoogleOAuth2LoginHandler(LoginHandler,
+            class GoogleOAuth2LoginHandler(tornado.web.RequestHandler,
                                            tornado.auth.GoogleOAuth2Mixin):
                 @tornado.gen.coroutine
                 def get(self):
@@ -990,7 +1029,7 @@ class GoogleOAuth2Mixin(OAuth2Mixin):
         })
 
         http.fetch(self._OAUTH_ACCESS_TOKEN_URL,
-                   self.async_callback(self._on_access_token, callback),
+                   functools.partial(self._on_access_token, callback),
                    method="POST", headers={'Content-Type': 'application/x-www-form-urlencoded'}, body=body)
 
     def _on_access_token(self, future, response):
@@ -1031,7 +1070,7 @@ class FacebookMixin(object):
             @tornado.web.asynchronous
             def get(self):
                 if self.get_argument("session", None):
-                    self.get_authenticated_user(self.async_callback(self._on_auth))
+                    self.get_authenticated_user(self._on_auth)
                     return
                 yield self.authenticate_redirect()
 
@@ -1117,7 +1156,7 @@ class FacebookMixin(object):
         session = escape.json_decode(self.get_argument("session"))
         self.facebook_request(
             method="facebook.users.getInfo",
-            callback=self.async_callback(
+            callback=functools.partial(
                 self._on_get_user_info, callback, session),
             session_key=session["session_key"],
             uids=session["uid"],
@@ -1143,7 +1182,7 @@ class FacebookMixin(object):
                 def get(self):
                     self.facebook_request(
                         method="stream.get",
-                        callback=self.async_callback(self._on_stream),
+                        callback=self._on_stream,
                         session_key=self.current_user["session_key"])
 
                 def _on_stream(self, stream):
@@ -1167,7 +1206,7 @@ class FacebookMixin(object):
         url = "http://api.facebook.com/restserver.php?" + \
             urllib_parse.urlencode(args)
         http = self.get_auth_http_client()
-        http.fetch(url, callback=self.async_callback(
+        http.fetch(url, callback=functools.partial(
             self._parse_response, callback))
 
     def _on_get_user_info(self, callback, session, users):
@@ -1265,7 +1304,7 @@ class FacebookGraphMixin(OAuth2Mixin):
             fields.update(extra_fields)
 
         http.fetch(self._oauth_request_token_url(**args),
-                   self.async_callback(self._on_access_token, redirect_uri, client_id,
+                   functools.partial(self._on_access_token, redirect_uri, client_id,
                                        client_secret, callback, fields))
 
     def _on_access_token(self, redirect_uri, client_id, client_secret,
@@ -1282,7 +1321,7 @@ class FacebookGraphMixin(OAuth2Mixin):
 
         self.facebook_request(
             path="/me",
-            callback=self.async_callback(
+            callback=functools.partial(
                 self._on_get_user_info, future, session, fields),
             access_token=session["access_token"],
             fields=",".join(fields)
@@ -1349,7 +1388,7 @@ class FacebookGraphMixin(OAuth2Mixin):
 
         if all_args:
             url += "?" + urllib_parse.urlencode(all_args)
-        callback = self.async_callback(self._on_facebook_request, callback)
+        callback = functools.partial(self._on_facebook_request, callback)
         http = self.get_auth_http_client()
         if post_args is not None:
             http.fetch(url, method="POST", body=urllib_parse.urlencode(post_args),

libs/tornado/http1connection.py

@@ -31,6 +31,26 @@ from tornado import stack_context
 from tornado.util import GzipDecompressor
 
 
+class _QuietException(Exception):
+    def __init__(self):
+        pass
+
+class _ExceptionLoggingContext(object):
+    """Used with the ``with`` statement when calling delegate methods to
+    log any exceptions with the given logger.  Any exceptions caught are
+    converted to _QuietException
+    """
+    def __init__(self, logger):
+        self.logger = logger
+
+    def __enter__(self):
+        pass
+
+    def __exit__(self, typ, value, tb):
+        if value is not None:
+            self.logger.error("Uncaught exception", exc_info=(typ, value, tb))
+            raise _QuietException
+
 class HTTP1ConnectionParameters(object):
     """Parameters for `.HTTP1Connection` and `.HTTP1ServerConnection`.
     """
@@ -155,9 +175,10 @@ class HTTP1Connection(httputil.HTTPConnection):
             self._disconnect_on_finish = not self._can_keep_alive(
                 start_line, headers)
             need_delegate_close = True
-            header_future = delegate.headers_received(start_line, headers)
+            with _ExceptionLoggingContext(app_log):
-            if header_future is not None:
+                header_future = delegate.headers_received(start_line, headers)
-                yield header_future
+                if header_future is not None:
+                    yield header_future
             if self.stream is None:
                 # We've been detached.
                 need_delegate_close = False
@@ -196,7 +217,8 @@ class HTTP1Connection(httputil.HTTPConnection):
             self._read_finished = True
             if not self._write_finished or self.is_client:
                 need_delegate_close = False
-                delegate.finish()
+                with _ExceptionLoggingContext(app_log):
+                    delegate.finish()
             # If we're waiting for the application to produce an asynchronous
             # response, and we're not detached, register a close callback
             # on the stream (we didn't need one while we were reading)
@@ -216,7 +238,8 @@ class HTTP1Connection(httputil.HTTPConnection):
             raise gen.Return(False)
         finally:
             if need_delegate_close:
-                delegate.on_connection_close()
+                with _ExceptionLoggingContext(app_log):
+                    delegate.on_connection_close()
             self._clear_callbacks()
         raise gen.Return(True)
 
@@ -478,7 +501,8 @@ class HTTP1Connection(httputil.HTTPConnection):
                 min(self.params.chunk_size, content_length), partial=True)
             content_length -= len(body)
             if not self._write_finished or self.is_client:
-                yield gen.maybe_future(delegate.data_received(body))
+                with _ExceptionLoggingContext(app_log):
+                    yield gen.maybe_future(delegate.data_received(body))
 
     @gen.coroutine
     def _read_chunked_body(self, delegate):
@@ -498,8 +522,8 @@ class HTTP1Connection(httputil.HTTPConnection):
                     min(bytes_to_read, self.params.chunk_size), partial=True)
                 bytes_to_read -= len(chunk)
                 if not self._write_finished or self.is_client:
-                    yield gen.maybe_future(
+                    with _ExceptionLoggingContext(app_log):
-                        delegate.data_received(chunk))
+                        yield gen.maybe_future(delegate.data_received(chunk))
             # chunk ends with \r\n
             crlf = yield self.stream.read_bytes(2)
             assert crlf == b"\r\n"
@@ -508,7 +532,8 @@ class HTTP1Connection(httputil.HTTPConnection):
     def _read_body_until_close(self, delegate):
         body = yield self.stream.read_until_close()
         if not self._write_finished or self.is_client:
-            delegate.data_received(body)
+            with _ExceptionLoggingContext(app_log):
+                delegate.data_received(body)
 
 
 class _GzipMessageDelegate(httputil.HTTPMessageDelegate):
@@ -610,11 +635,12 @@ class HTTP1ServerConnection(object):
                 except (iostream.StreamClosedError,
                         iostream.UnsatisfiableReadError):
                     return
+                except _QuietException:
+                    # This exception was already logged.
+                    conn.close()
+                    return
                 except Exception:
-                    # TODO: this is probably too broad; it would be better to
+                    gen_log.error("Uncaught exception", exc_info=True)
-                    # wrap all delegate calls in something that writes to app_log,
-                    # and then errors that reach this point can be gen_log.
-                    app_log.error("Uncaught exception", exc_info=True)
                     conn.close()
                     return
                 if not ret:

libs/tornado/ioloop.py

@@ -32,6 +32,7 @@ import datetime
 import errno
 import functools
 import heapq
+import itertools
 import logging
 import numbers
 import os
@@ -585,6 +586,7 @@ class PollIOLoop(IOLoop):
         self._closing = False
         self._thread_ident = None
         self._blocking_signal_threshold = None
+        self._timeout_counter = itertools.count()
         
         # Create a pipe that we send bogus data to when we want to wake
         # the I/O loop when it is idle
@@ -835,7 +837,7 @@ class _Timeout(object):
     """An IOLoop timeout, a UNIX timestamp and a callback"""
 
     # Reduce memory overhead when there are lots of pending callbacks
-    __slots__ = ['deadline', 'callback']
+    __slots__ = ['deadline', 'callback', 'tiebreaker']
 
     def __init__(self, deadline, callback, io_loop):
         if isinstance(deadline, numbers.Real):
@@ -849,6 +851,7 @@ class _Timeout(object):
         else:
             raise TypeError("Unsupported deadline %r" % deadline)
         self.callback = callback
+        self.tiebreaker = next(io_loop._timeout_counter)
 
     @staticmethod
     def timedelta_to_seconds(td):
@@ -860,12 +863,12 @@ class _Timeout(object):
     # in python2.5, and __lt__ in 2.6+ (sort() and most other comparisons
     # use __lt__).
     def __lt__(self, other):
-        return ((self.deadline, id(self)) <
+        return ((self.deadline, self.tiebreaker) <
-                (other.deadline, id(other)))
+                (other.deadline, other.tiebreaker))
 
     def __le__(self, other):
-        return ((self.deadline, id(self)) <=
+        return ((self.deadline, self.tiebreaker) <=
-                (other.deadline, id(other)))
+                (other.deadline, other.tiebreaker))
 
 
 class PeriodicCallback(object):

libs/tornado/util.py

@@ -12,11 +12,19 @@ and `.Resolver`.
 
 from __future__ import absolute_import, division, print_function, with_statement
 
+import array
 import inspect
+import os
 import sys
 import zlib
 
 
+try:
+    xrange  # py2
+except NameError:
+    xrange = range  # py3
+
+
 class ObjectDict(dict):
     """Makes a dictionary behave like an object, with attribute-style access.
     """
@@ -303,6 +311,41 @@ class ArgReplacer(object):
         return old_value, args, kwargs
 
 
+def _websocket_mask_python(mask, data):
+    """Websocket masking function.
+
+    `mask` is a `bytes` object of length 4; `data` is a `bytes` object of any length.
+    Returns a `bytes` object of the same length as `data` with the mask applied
+    as specified in section 5.3 of RFC 6455.
+
+    This pure-python implementation may be replaced by an optimized version when available.
+    """
+    mask = array.array("B", mask)
+    unmasked = array.array("B", data)
+    for i in xrange(len(data)):
+        unmasked[i] = unmasked[i] ^ mask[i % 4]
+    if hasattr(unmasked, 'tobytes'):
+        # tostring was deprecated in py32.  It hasn't been removed,
+        # but since we turn on deprecation warnings in our tests
+        # we need to use the right one.
+        return unmasked.tobytes()
+    else:
+        return unmasked.tostring()
+
+if (os.environ.get('TORNADO_NO_EXTENSION') or
+    os.environ.get('TORNADO_EXTENSION') == '0'):
+    # These environment variables exist to make it easier to do performance
+    # comparisons; they are not guaranteed to remain supported in the future.
+    _websocket_mask = _websocket_mask_python
+else:
+    try:
+        from tornado.speedups import websocket_mask as _websocket_mask
+    except ImportError:
+        if os.environ.get('TORNADO_EXTENSION') == '1':
+            raise
+        _websocket_mask = _websocket_mask_python
+
+
 def doctests():
     import doctest
     return doctest.DocTestSuite()

libs/tornado/web.py

@@ -72,7 +72,6 @@ import time
 import tornado
 import traceback
 import types
-import uuid
 
 from tornado.concurrent import Future, is_future
 from tornado import escape
@@ -84,7 +83,7 @@ from tornado.log import access_log, app_log, gen_log
 from tornado import stack_context
 from tornado import template
 from tornado.escape import utf8, _unicode
-from tornado.util import bytes_type, import_object, ObjectDict, raise_exc_info, unicode_type
+from tornado.util import bytes_type, import_object, ObjectDict, raise_exc_info, unicode_type, _websocket_mask
 
 try:
     from io import BytesIO  # python 3
@@ -1076,16 +1075,87 @@ class RequestHandler(object):
         as a potential forgery.
 
         See http://en.wikipedia.org/wiki/Cross-site_request_forgery
+
+        .. versionchanged:: 3.2.2
+           The xsrf token will now be have a random mask applied in every
+           request, which makes it safe to include the token in pages
+           that are compressed.  See http://breachattack.com for more
+           information on the issue fixed by this change.  Old (version 1)
+           cookies will be converted to version 2 when this method is called
+           unless the ``xsrf_cookie_version`` `Application` setting is
+           set to 1.
         """
         if not hasattr(self, "_xsrf_token"):
-            token = self.get_cookie("_xsrf")
+            version, token, timestamp = self._get_raw_xsrf_token()
-            if not token:
+            output_version = self.settings.get("xsrf_cookie_version", 2)
-                token = binascii.b2a_hex(uuid.uuid4().bytes)
+            if output_version == 1:
+                self._xsrf_token = binascii.b2a_hex(token)
+            elif output_version == 2:
+                mask = os.urandom(4)
+                self._xsrf_token = b"|".join([
+                    b"2",
+                    binascii.b2a_hex(mask),
+                    binascii.b2a_hex(_websocket_mask(mask, token)),
+                    utf8(str(int(timestamp)))])
+            else:
+                raise ValueError("unknown xsrf cookie version %d",
+                                 output_version)
+            if version is None:
                 expires_days = 30 if self.current_user else None
-                self.set_cookie("_xsrf", token, expires_days=expires_days)
+                self.set_cookie("_xsrf", self._xsrf_token,
-            self._xsrf_token = token
+                                expires_days=expires_days)
         return self._xsrf_token
 
+    def _get_raw_xsrf_token(self):
+        """Read or generate the xsrf token in its raw form.
+
+        The raw_xsrf_token is a tuple containing:
+
+        * version: the version of the cookie from which this token was read,
+          or None if we generated a new token in this request.
+        * token: the raw token data; random (non-ascii) bytes.
+        * timestamp: the time this token was generated (will not be accurate
+          for version 1 cookies)
+        """
+        if not hasattr(self, '_raw_xsrf_token'):
+            cookie = self.get_cookie("_xsrf")
+            if cookie:
+                version, token, timestamp = self._decode_xsrf_token(cookie)
+            else:
+                version, token, timestamp = None, None, None
+            if token is None:
+                version = None
+                token = os.urandom(16)
+                timestamp = time.time()
+            self._raw_xsrf_token = (version, token, timestamp)
+        return self._raw_xsrf_token
+
+    def _decode_xsrf_token(self, cookie):
+        """Convert a cookie string into a the tuple form returned by
+        _get_raw_xsrf_token.
+        """
+        m = _signed_value_version_re.match(utf8(cookie))
+        if m:
+            version = int(m.group(1))
+            if version == 2:
+                _, mask, masked_token, timestamp = cookie.split("|")
+                mask = binascii.a2b_hex(utf8(mask))
+                token = _websocket_mask(
+                    mask, binascii.a2b_hex(utf8(masked_token)))
+                timestamp = int(timestamp)
+                return version, token, timestamp
+            else:
+                # Treat unknown versions as not present instead of failing.
+                return None, None, None
+        elif len(cookie) == 32:
+            version = 1
+            token = binascii.a2b_hex(utf8(cookie))
+            # We don't have a usable timestamp in older versions.
+            timestamp = int(time.time())
+            return (version, token, timestamp)
+        else:
+            return None, None, None
+
     def check_xsrf_cookie(self):
         """Verifies that the ``_xsrf`` cookie matches the ``_xsrf`` argument.
 
@@ -1106,13 +1176,19 @@ class RequestHandler(object):
         information please see
         http://www.djangoproject.com/weblog/2011/feb/08/security/
         http://weblog.rubyonrails.org/2011/2/8/csrf-protection-bypass-in-ruby-on-rails
+
+        .. versionchanged:: 3.2.2
+           Added support for cookie version 2.  Both versions 1 and 2 are
+           supported.
         """
         token = (self.get_argument("_xsrf", None) or
                  self.request.headers.get("X-Xsrftoken") or
                  self.request.headers.get("X-Csrftoken"))
         if not token:
             raise HTTPError(403, "'_xsrf' argument missing from POST")
-        if self.xsrf_token != token:
+        _, token, _ = self._decode_xsrf_token(token)
+        _, expected_token, _ = self._get_raw_xsrf_token()
+        if not _time_independent_equals(utf8(token), utf8(expected_token)):
             raise HTTPError(403, "XSRF cookie does not match POST argument")
 
     def xsrf_form_html(self):

libs/tornado/websocket.py

@@ -20,7 +20,6 @@ communication between the browser and server.
 from __future__ import absolute_import, division, print_function, with_statement
 # Author: Jacob Kristhammar, 2010
 
-import array
 import base64
 import collections
 import functools
@@ -39,7 +38,7 @@ from tornado.iostream import StreamClosedError
 from tornado.log import gen_log, app_log
 from tornado import simple_httpclient
 from tornado.tcpclient import TCPClient
-from tornado.util import bytes_type, unicode_type
+from tornado.util import bytes_type, unicode_type, _websocket_mask
 
 try:
     from urllib.parse import urlparse # py2
@@ -988,38 +987,3 @@ def websocket_connect(url, io_loop=None, callback=None, connect_timeout=None):
     if callback is not None:
         io_loop.add_future(conn.connect_future, callback)
     return conn.connect_future
-
-
-def _websocket_mask_python(mask, data):
-    """Websocket masking function.
-
-    `mask` is a `bytes` object of length 4; `data` is a `bytes` object of any length.
-    Returns a `bytes` object of the same length as `data` with the mask applied
-    as specified in section 5.3 of RFC 6455.
-
-    This pure-python implementation may be replaced by an optimized version when available.
-    """
-    mask = array.array("B", mask)
-    unmasked = array.array("B", data)
-    for i in xrange(len(data)):
-        unmasked[i] = unmasked[i] ^ mask[i % 4]
-    if hasattr(unmasked, 'tobytes'):
-        # tostring was deprecated in py32.  It hasn't been removed,
-        # but since we turn on deprecation warnings in our tests
-        # we need to use the right one.
-        return unmasked.tobytes()
-    else:
-        return unmasked.tostring()
-
-if (os.environ.get('TORNADO_NO_EXTENSION') or
-        os.environ.get('TORNADO_EXTENSION') == '0'):
-    # These environment variables exist to make it easier to do performance
-    # comparisons; they are not guaranteed to remain supported in the future.
-    _websocket_mask = _websocket_mask_python
-else:
-    try:
-        from tornado.speedups import websocket_mask as _websocket_mask
-    except ImportError:
-        if os.environ.get('TORNADO_EXTENSION') == '1':
-            raise
-        _websocket_mask = _websocket_mask_python