Merge branch 'develop'

CouchPotato.py

@@ -10,7 +10,6 @@ import socket
 import subprocess
 import sys
 import traceback
-import time
 
 # Root path
 base_path = dirname(os.path.abspath(__file__))

couchpotato/core/helpers/variable.py

@@ -380,3 +380,11 @@ def getFreeSpace(directories):
         free_space[folder] = size
 
     return free_space
+
+
+def find(func, iterable):
+    for item in iterable:
+        if func(item):
+            return item
+
+    return None

couchpotato/core/media/__init__.py

@@ -65,10 +65,13 @@ class MediaBase(Plugin):
 
         return def_title or 'UNKNOWN'
 
-    def getPoster(self, image_urls, existing_files):
-        image_type = 'poster'
+    def getPoster(self, media, image_urls):
+        if 'files' not in media:
+            media['files'] = {}
 
-        # Remove non-existing files
+        existing_files = media['files']
+
+        image_type = 'poster'
         file_type = 'image_%s' % image_type
 
         # Make existing unique

couchpotato/core/media/_base/library/main.py

@@ -1,10 +1,47 @@
+from couchpotato import get_db
+from couchpotato.api import addApiView
 from couchpotato.core.event import addEvent, fireEvent
+from couchpotato.core.logger import CPLog
 from couchpotato.core.media._base.library.base import LibraryBase
 
+log = CPLog(__name__)
+
 
 class Library(LibraryBase):
     def __init__(self):
         addEvent('library.title', self.title)
+        addEvent('library.related', self.related)
+        addEvent('library.tree', self.tree)
+
+        addEvent('library.root', self.root)
+
+        addApiView('library.query', self.queryView)
+        addApiView('library.related', self.relatedView)
+        addApiView('library.tree', self.treeView)
+
+    def queryView(self, media_id, **kwargs):
+        db = get_db()
+        media = db.get('id', media_id)
+
+        return {
+            'result': fireEvent('library.query', media, single = True)
+        }
+
+    def relatedView(self, media_id, **kwargs):
+        db = get_db()
+        media = db.get('id', media_id)
+
+        return {
+            'result': fireEvent('library.related', media, single = True)
+        }
+
+    def treeView(self, media_id, **kwargs):
+        db = get_db()
+        media = db.get('id', media_id)
+
+        return {
+            'result': fireEvent('library.tree', media, single = True)
+        }
 
     def title(self, library):
         return fireEvent(
@@ -16,3 +53,76 @@ class Library(LibraryBase):
             include_identifier = False,
             single = True
         )
+
+    def related(self, media):
+        result = {self.key(media['type']): media}
+
+        db = get_db()
+        cur = media
+
+        while cur and cur.get('parent_id'):
+            cur = db.get('id', cur['parent_id'])
+
+            result[self.key(cur['type'])] = cur
+
+        children = db.get_many('media_children', media['_id'], with_doc = True)
+
+        for item in children:
+            key = self.key(item['doc']['type']) + 's'
+
+            if key not in result:
+                result[key] = []
+
+            result[key].append(item['doc'])
+
+        return result
+
+    def root(self, media):
+        db = get_db()
+        cur = media
+
+        while cur and cur.get('parent_id'):
+            cur = db.get('id', cur['parent_id'])
+
+        return cur
+
+    def tree(self, media = None, media_id = None):
+        db = get_db()
+
+        if media:
+            result = media
+        elif media_id:
+            result = db.get('id', media_id, with_doc = True)
+        else:
+            return None
+
+        # Find children
+        items = db.get_many('media_children', result['_id'], with_doc = True)
+        keys = []
+
+        # Build children arrays
+        for item in items:
+            key = self.key(item['doc']['type']) + 's'
+
+            if key not in result:
+                result[key] = {}
+            elif type(result[key]) is not dict:
+                result[key] = {}
+
+            if key not in keys:
+                keys.append(key)
+
+            result[key][item['_id']] = fireEvent('library.tree', item['doc'], single = True)
+
+        # Unique children
+        for key in keys:
+            result[key] = result[key].values()
+
+        # Include releases
+        result['releases'] = fireEvent('release.for_media', result['_id'], single = True)
+
+        return result
+
+    def key(self, media_type):
+        parts = media_type.split('.')
+        return parts[-1]

couchpotato/core/media/_base/matcher/main.py

@@ -40,7 +40,7 @@ class Matcher(MatcherBase):
         return False
 
     def correctTitle(self, chain, media):
-        root_library = media['library']['root_library']
+        root = fireEvent('library.root', media, single = True)
 
         if 'show_name' not in chain.info or not len(chain.info['show_name']):
             log.info('Wrong: missing show name in parsed result')
@@ -50,10 +50,10 @@ class Matcher(MatcherBase):
         chain_words = [x.lower() for x in chain.info['show_name']]
 
         # Build a list of possible titles of the media we are searching for
-        titles = root_library['info']['titles']
+        titles = root['info']['titles']
 
         # Add year suffix titles (will result in ['<name_one>', '<name_one> <suffix_one>', '<name_two>', ...])
-        suffixes = [None, root_library['info']['year']]
+        suffixes = [None, root['info']['year']]
 
         titles = [
             title + ((' %s' % suffix) if suffix else '')

couchpotato/core/media/_base/media/main.py

@@ -44,15 +44,15 @@ class MediaPlugin(MediaBase):
             'desc': 'List media',
             'params': {
                 'type': {'type': 'string', 'desc': 'Media type to filter on.'},
-                'status': {'type': 'array or csv', 'desc': 'Filter movie by status. Example:"active,done"'},
-                'release_status': {'type': 'array or csv', 'desc': 'Filter movie by status of its releases. Example:"snatched,available"'},
-                'limit_offset': {'desc': 'Limit and offset the movie list. Examples: "50" or "50,30"'},
-                'starts_with': {'desc': 'Starts with these characters. Example: "a" returns all movies starting with the letter "a"'},
-                'search': {'desc': 'Search movie title'},
+                'status': {'type': 'array or csv', 'desc': 'Filter media by status. Example:"active,done"'},
+                'release_status': {'type': 'array or csv', 'desc': 'Filter media by status of its releases. Example:"snatched,available"'},
+                'limit_offset': {'desc': 'Limit and offset the media list. Examples: "50" or "50,30"'},
+                'starts_with': {'desc': 'Starts with these characters. Example: "a" returns all media starting with the letter "a"'},
+                'search': {'desc': 'Search media title'},
             },
             'return': {'type': 'object', 'example': """{
     'success': True,
-    'empty': bool, any movies returned or not,
+    'empty': bool, any media returned or not,
     'media': array, media found,
 }"""}
         })
@@ -109,7 +109,7 @@ class MediaPlugin(MediaBase):
 
         try:
             media = get_db().get('id', media_id)
-            event = '%s.update_info' % media.get('type')
+            event = '%s.update' % media.get('type')
 
             def handler():
                 fireEvent(event, media_id = media_id, on_complete = self.createOnComplete(media_id))
@@ -160,10 +160,13 @@ class MediaPlugin(MediaBase):
             'media': media,
         }
 
-    def withStatus(self, status, with_doc = True):
+    def withStatus(self, status, types = None, with_doc = True):
 
         db = get_db()
 
+        if types and not isinstance(types, (list, tuple)):
+            types = [types]
+
         status = list(status if isinstance(status, (list, tuple)) else [status])
 
         for s in status:
@@ -171,6 +174,10 @@ class MediaPlugin(MediaBase):
                 if with_doc:
                     try:
                         doc = db.get('id', ms['_id'])
+
+                        if types and doc.get('type') not in types:
+                            continue
+
                         yield doc
                     except RecordNotFound:
                         log.debug('Record not found, skipping: %s', ms['_id'])
@@ -178,17 +185,15 @@ class MediaPlugin(MediaBase):
                     yield ms
 
     def withIdentifiers(self, identifiers, with_doc = False):
-
         db = get_db()
 
         for x in identifiers:
             try:
-                media = db.get('media', '%s-%s' % (x, identifiers[x]), with_doc = with_doc)
-                return media
+                return db.get('media', '%s-%s' % (x, identifiers[x]), with_doc = with_doc)
             except:
                 pass
 
-        log.debug('No media found with identifiers: %s', identifiers)
+        return False
 
     def list(self, types = None, status = None, release_status = None, status_or = False, limit_offset = None, with_tags = None, starts_with = None, search = None):
 
@@ -307,9 +312,22 @@ class MediaPlugin(MediaBase):
     def addSingleListView(self):
 
         for media_type in fireEvent('media.types', merge = True):
-            def tempList(*args, **kwargs):
-                return self.listView(types = media_type, **kwargs)
-            addApiView('%s.list' % media_type, tempList)
+            tempList = lambda media_type = media_type, *args, **kwargs : self.listView(type = media_type, **kwargs)
+            addApiView('%s.list' % media_type, tempList, docs = {
+                'desc': 'List media',
+                'params': {
+                    'status': {'type': 'array or csv', 'desc': 'Filter ' + media_type + ' by status. Example:"active,done"'},
+                    'release_status': {'type': 'array or csv', 'desc': 'Filter ' + media_type + ' by status of its releases. Example:"snatched,available"'},
+                    'limit_offset': {'desc': 'Limit and offset the ' + media_type + ' list. Examples: "50" or "50,30"'},
+                    'starts_with': {'desc': 'Starts with these characters. Example: "a" returns all ' + media_type + 's starting with the letter "a"'},
+                    'search': {'desc': 'Search ' + media_type + ' title'},
+                },
+                'return': {'type': 'object', 'example': """{
+        'success': True,
+        'empty': bool, any """ + media_type + """s returned or not,
+        'media': array, media found,
+    }"""}
+            })
 
     def availableChars(self, types = None, status = None, release_status = None):
 
@@ -376,8 +394,7 @@ class MediaPlugin(MediaBase):
     def addSingleCharView(self):
 
         for media_type in fireEvent('media.types', merge = True):
-            def tempChar(*args, **kwargs):
-                return self.charView(types = media_type, **kwargs)
+            tempChar = lambda media_type = media_type, *args, **kwargs : self.charView(type = media_type, **kwargs)
             addApiView('%s.available_chars' % media_type, tempChar)
 
     def delete(self, media_id, delete_from = None):
@@ -446,9 +463,14 @@ class MediaPlugin(MediaBase):
     def addSingleDeleteView(self):
 
         for media_type in fireEvent('media.types', merge = True):
-            def tempDelete(*args, **kwargs):
-                return self.deleteView(types = media_type, *args, **kwargs)
-            addApiView('%s.delete' % media_type, tempDelete)
+            tempDelete = lambda media_type = media_type, *args, **kwargs : self.deleteView(type = media_type, **kwargs)
+            addApiView('%s.delete' % media_type, tempDelete, docs = {
+            'desc': 'Delete a ' + media_type + ' from the wanted list',
+            'params': {
+                'id': {'desc': 'Media ID(s) you want to delete.', 'type': 'int (comma separated)'},
+                'delete_from': {'desc': 'Delete ' + media_type + ' from this page', 'type': 'string: all (default), wanted, manage'},
+            }
+        })
 
     def restatus(self, media_id):
 
@@ -470,12 +492,13 @@ class MediaPlugin(MediaBase):
                     done_releases = [release for release in media_releases if release.get('status') == 'done']
 
                     if done_releases:
-                        # Only look at latest added release
-                        release = sorted(done_releases, key = itemgetter('last_edit'), reverse = True)[0]
 
                         # Check if we are finished with the media
-                        if fireEvent('quality.isfinish', {'identifier': release['quality'], 'is_3d': release.get('is_3d', False)}, profile, timedelta(seconds = time.time() - release['last_edit']).days, single = True):
-                            m['status'] = 'done'
+                        for release in done_releases:
+                            if fireEvent('quality.isfinish', {'identifier': release['quality'], 'is_3d': release.get('is_3d', False)}, profile, timedelta(seconds = time.time() - release['last_edit']).days, single = True):
+                                m['status'] = 'done'
+                                break
+
                     elif previous_status == 'done':
                         m['status'] = 'done'
 

couchpotato/core/media/_base/providers/torrent/bitsoup.py

@@ -22,6 +22,9 @@ class Base(TorrentProvider):
     http_time_between_calls = 1  # Seconds
     only_tables_tags = SoupStrainer('table')
 
+    torrent_name_cell = 1
+    torrent_download_cell = 2
+
     def _searchOnTitle(self, title, movie, quality, results):
 
         url = self.urls['search'] % self.buildUrl(title, movie, quality)
@@ -40,8 +43,8 @@ class Base(TorrentProvider):
 
                     all_cells = result.find_all('td')
 
-                    torrent = all_cells[1].find('a')
-                    download = all_cells[3].find('a')
+                    torrent = all_cells[self.torrent_name_cell].find('a')
+                    download = all_cells[self.torrent_download_cell].find('a')
 
                     torrent_id = torrent['href']
                     torrent_id = torrent_id.replace('details.php?id=', '')

couchpotato/core/media/_base/providers/torrent/kickasstorrents.py

@@ -34,8 +34,7 @@ class Base(TorrentMagnetProvider):
         'http://kickass.pw',
         'http://kickassto.come.in',
         'http://katproxy.ws',
-        'http://www.kickassunblock.info',
-        'http://www.kickassproxy.info',
+        'http://kickass.bitproxy.eu',
         'http://katph.eu',
         'http://kickassto.come.in',
     ]

couchpotato/core/media/_base/providers/torrent/thepiratebay.py

@@ -24,7 +24,7 @@ class Base(TorrentMagnetProvider):
     http_time_between_calls = 0
 
     proxy_list = [
-        'https://www.dieroschtibay.org',
+        'https://dieroschtibay.org',
         'https://thebay.al',
         'https://thepiratebay.se',
         'http://thepiratebay.se.net',

couchpotato/core/media/_base/providers/torrent/torrentleech.py

@@ -13,12 +13,12 @@ log = CPLog(__name__)
 class Base(TorrentProvider):
 
     urls = {
-        'test': 'http://www.torrentleech.org/',
-        'login': 'http://www.torrentleech.org/user/account/login/',
-        'login_check': 'http://torrentleech.org/user/messages',
-        'detail': 'http://www.torrentleech.org/torrent/%s',
-        'search': 'http://www.torrentleech.org/torrents/browse/index/query/%s/categories/%d',
-        'download': 'http://www.torrentleech.org%s',
+        'test': 'https://www.torrentleech.org/',
+        'login': 'https://www.torrentleech.org/user/account/login/',
+        'login_check': 'https://torrentleech.org/user/messages',
+        'detail': 'https://www.torrentleech.org/torrent/%s',
+        'search': 'https://www.torrentleech.org/torrents/browse/index/query/%s/categories/%d',
+        'download': 'https://www.torrentleech.org%s',
     }
 
     http_time_between_calls = 1  # Seconds

couchpotato/core/media/movie/_base/main.py

@@ -46,7 +46,7 @@ class MovieBase(MovieTypeBase):
         })
 
         addEvent('movie.add', self.add)
-        addEvent('movie.update_info', self.updateInfo)
+        addEvent('movie.update', self.update)
         addEvent('movie.update_release_dates', self.updateReleaseDate)
 
     def add(self, params = None, force_readd = True, search_after = True, update_after = True, notify_after = True, status = None):
@@ -172,7 +172,7 @@ class MovieBase(MovieTypeBase):
             # Trigger update info
             if added and update_after:
                 # Do full update to get images etc
-                fireEventAsync('movie.update_info', m['_id'], default_title = params.get('title'), on_complete = onComplete)
+                fireEventAsync('movie.update', m['_id'], default_title = params.get('title'), on_complete = onComplete)
 
             # Remove releases
             for rel in fireEvent('release.for_media', m['_id'], single = True):
@@ -256,7 +256,7 @@ class MovieBase(MovieTypeBase):
             'success': False,
         }
 
-    def updateInfo(self, media_id = None, identifier = None, default_title = None, extended = False):
+    def update(self, media_id = None, identifier = None, default_title = None, extended = False):
         """
         Update movie information inside media['doc']['info']
 
@@ -312,37 +312,11 @@ class MovieBase(MovieTypeBase):
                 media['title'] = def_title
 
             # Files
-            images = info.get('images', [])
-            media['files'] = media.get('files', {})
-            for image_type in ['poster']:
+            image_urls = info.get('images', [])
 
-                # Remove non-existing files
-                file_type = 'image_%s' % image_type
-                existing_files = list(set(media['files'].get(file_type, [])))
-                for ef in media['files'].get(file_type, []):
-                    if not os.path.isfile(ef):
-                        existing_files.remove(ef)
-
-                # Replace new files list
-                media['files'][file_type] = existing_files
-                if len(existing_files) == 0:
-                    del media['files'][file_type]
-
-                # Loop over type
-                for image in images.get(image_type, []):
-                    if not isinstance(image, (str, unicode)):
-                        continue
-
-                    if file_type not in media['files'] or len(media['files'].get(file_type, [])) == 0:
-                        file_path = fireEvent('file.download', url = image, single = True)
-                        if file_path:
-                            media['files'][file_type] = [file_path]
-                            break
-                    else:
-                        break
+            self.getPoster(media, image_urls)
 
             db.update(media)
-
             return media
         except:
             log.error('Failed update media: %s', traceback.format_exc())
@@ -363,7 +337,7 @@ class MovieBase(MovieTypeBase):
             media = db.get('id', media_id)
 
             if not media.get('info'):
-                media = self.updateInfo(media_id)
+                media = self.update(media_id)
                 dates = media.get('info', {}).get('release_date')
             else:
                 dates = media.get('info').get('release_date')

couchpotato/core/media/movie/_base/static/movie.js

@@ -159,7 +159,7 @@ var Movie = new Class({
 					}
 				}
 			}),
-			self.thumbnail = (self.data.files && self.data.files.image_poster) ? new Element('img', {
+			self.thumbnail = (self.data.files && self.data.files.image_poster && self.data.files.image_poster.length > 0) ? new Element('img', {
 				'class': 'type_image poster',
 				'src': Api.createUrl('file.cache') + self.data.files.image_poster[0].split(Api.getOption('path_sep')).pop()
 			}): null,

couchpotato/core/media/movie/providers/automation/bluray.py

@@ -1,3 +1,5 @@
+import traceback
+
 from bs4 import BeautifulSoup
 from couchpotato import fireEvent
 from couchpotato.core.helpers.rss import RSS
@@ -5,6 +7,7 @@ from couchpotato.core.helpers.variable import tryInt
 from couchpotato.core.logger import CPLog
 from couchpotato.core.media.movie.providers.automation.base import Automation
 
+
 log = CPLog(__name__)
 
 autoload = 'Bluray'
@@ -34,27 +37,49 @@ class Bluray(Automation, RSS):
 
                 try:
                     # Stop if the release year is before the minimal year
-                    page_year = soup.body.find_all('center')[3].table.tr.find_all('td', recursive = False)[3].h3.get_text().split(', ')[1]
-                    if tryInt(page_year) < self.getMinimal('year'):
+                    brk = False
+                    h3s = soup.body.find_all('h3')
+                    for h3 in h3s:
+                        if h3.parent.name != 'a':
+
+                            try:
+                                page_year = tryInt(h3.get_text()[-4:])
+                                if page_year > 0 and page_year < self.getMinimal('year'):
+                                    brk = True
+                            except:
+                                log.error('Failed determining page year: %s', traceback.format_exc())
+                                brk = True
+                                break
+
+                    if brk:
                         break
 
-                    for table in soup.body.find_all('center')[3].table.tr.find_all('td', recursive = False)[3].find_all('table')[1:20]:
-                        name = table.h3.get_text().lower().split('blu-ray')[0].strip()
-                        year = table.small.get_text().split('|')[1].strip()
+                    for h3 in h3s:
+                        try:
+                            if h3.parent.name == 'a':
+                                name = h3.get_text().lower().split('blu-ray')[0].strip()
 
-                        if not name.find('/') == -1:  # make sure it is not a double movie release
-                            continue
+                                if not name.find('/') == -1:  # make sure it is not a double movie release
+                                    continue
 
-                        if tryInt(year) < self.getMinimal('year'):
-                            continue
+                                if not h3.parent.parent.small:  # ignore non-movie tables
+                                    continue
 
-                        imdb = self.search(name, year)
+                                year = h3.parent.parent.small.get_text().split('|')[1].strip()
 
-                        if imdb:
-                            if self.isMinimalMovie(imdb):
-                                movies.append(imdb['imdb'])
+                                if tryInt(year) < self.getMinimal('year'):
+                                    continue
+
+                                imdb = self.search(name, year)
+
+                                if imdb:
+                                    if self.isMinimalMovie(imdb):
+                                        movies.append(imdb['imdb'])
+                        except:
+                            log.debug('Error parsing movie html: %s', traceback.format_exc())
+                            break
                 except:
-                    log.debug('Error loading page: %s', page)
+                    log.debug('Error loading page %s: %s', (page, traceback.format_exc()))
                     break
 
             self.conf('backlog', value = False)
@@ -134,7 +159,7 @@ config = [{
                 {
                     'name': 'backlog',
                     'advanced': True,
-                    'description': 'Parses the history until the minimum movie year is reached. (Will be disabled once it has completed)',
+                    'description': ('Parses the history until the minimum movie year is reached. (Takes a while)', 'Will be disabled once it has completed'),
                     'default': False,
                     'type': 'bool',
                 },

couchpotato/core/media/movie/providers/metadata/base.py

@@ -28,7 +28,7 @@ class MovieMetaData(MetaDataBase):
 
         # Update library to get latest info
         try:
-            group['media'] = fireEvent('movie.update_info', group['media'].get('_id'), identifier = getIdentifier(group['media']), extended = True, single = True)
+            group['media'] = fireEvent('movie.update', group['media'].get('_id'), identifier = getIdentifier(group['media']), extended = True, single = True)
         except:
             log.error('Failed to update movie, before creating metadata: %s', traceback.format_exc())
 

couchpotato/core/media/movie/searcher.py

@@ -74,7 +74,7 @@ class MovieSearcher(SearcherBase, MovieTypeBase):
         self.in_progress = True
         fireEvent('notify.frontend', type = 'movie.searcher.started', data = True, message = 'Full search started')
 
-        medias = [x['_id'] for x in fireEvent('media.with_status', 'active', with_doc = False, single = True)]
+        medias = [x['_id'] for x in fireEvent('media.with_status', 'active', 'movie', single = True)]
         random.shuffle(medias)
 
         total = len(medias)
@@ -94,7 +94,7 @@ class MovieSearcher(SearcherBase, MovieTypeBase):
                     self.single(media, search_protocols, manual = manual)
                 except IndexError:
                     log.error('Forcing library update for %s, if you see this often, please report: %s', (getIdentifier(media), traceback.format_exc()))
-                    fireEvent('movie.update_info', media_id)
+                    fireEvent('movie.update', media_id)
                 except:
                     log.error('Search failed for %s: %s', (getIdentifier(media), traceback.format_exc()))
 

couchpotato/core/media/movie/suggestion/main.py

@@ -27,7 +27,7 @@ class Suggestion(Plugin):
         else:
 
             if not movies or len(movies) == 0:
-                active_movies = fireEvent('media.with_status', ['active', 'done'], single = True)
+                active_movies = fireEvent('media.with_status', ['active', 'done'], 'movie', single = True)
                 movies = [getIdentifier(x) for x in active_movies]
 
             if not ignored or len(ignored) == 0:

couchpotato/core/notifications/core/main.py

@@ -66,7 +66,9 @@ class CoreNotifier(Notification):
         fireEvent('schedule.interval', 'core.clean_messages', self.cleanMessages, seconds = 15, single = True)
 
         addEvent('app.load', self.clean)
-        addEvent('app.load', self.checkMessages)
+
+        if not Env.get('dev'):
+            addEvent('app.load', self.checkMessages)
 
         self.messages = []
         self.listeners = []

couchpotato/core/notifications/xbmc.py

@@ -7,8 +7,8 @@ import urllib
 from couchpotato.core.helpers.variable import splitString, getTitle
 from couchpotato.core.logger import CPLog
 from couchpotato.core.notifications.base import Notification
-import requests
-from requests.packages.urllib3.exceptions import MaxRetryError, ConnectionError
+from requests.exceptions import ConnectionError, Timeout
+from requests.packages.urllib3.exceptions import MaxRetryError
 
 
 log = CPLog(__name__)
@@ -172,7 +172,7 @@ class XBMC(Notification):
                 # manually fake expected response array
                 return [{'result': 'Error'}]
 
-        except (MaxRetryError, requests.exceptions.Timeout, ConnectionError):
+        except (MaxRetryError, Timeout, ConnectionError):
             log.info2('Couldn\'t send request to XBMC, assuming it\'s turned off')
             return [{'result': 'Error'}]
         except:
@@ -208,7 +208,7 @@ class XBMC(Notification):
             log.debug('Returned from request %s: %s', (host, response))
 
             return response
-        except (MaxRetryError, requests.exceptions.Timeout, ConnectionError):
+        except (MaxRetryError, Timeout, ConnectionError):
             log.info2('Couldn\'t send request to XBMC, assuming it\'s turned off')
             return []
         except:

couchpotato/core/plugins/dashboard.py

@@ -1,4 +1,3 @@
-from datetime import date
 import random as rndm
 import time
 
@@ -48,7 +47,6 @@ class Dashboard(Plugin):
         active_ids = [x['_id'] for x in fireEvent('media.with_status', 'active', with_doc = False, single = True)]
 
         medias = []
-        now_year = date.today().year
 
         if len(active_ids) > 0:
 
@@ -62,7 +60,7 @@ class Dashboard(Plugin):
             for media_id in active_ids:
                 media = db.get('id', media_id)
 
-                pp = profile_pre.get(media['profile_id'])
+                pp = profile_pre.get(media.get('profile_id'))
                 if not pp: continue
 
                 eta = media['info'].get('release_date', {}) or {}
@@ -70,22 +68,25 @@ class Dashboard(Plugin):
 
                 # Theater quality
                 if pp.get('theater') and fireEvent('movie.searcher.could_be_released', True, eta, media['info']['year'], single = True):
-                    coming_soon = True
+                    coming_soon = 'theater'
                 elif pp.get('dvd') and fireEvent('movie.searcher.could_be_released', False, eta, media['info']['year'], single = True):
-                    coming_soon = True
+                    coming_soon = 'dvd'
 
                 if coming_soon:
 
                     # Don't list older movies
-                    if ((not late and (media['info']['year'] >= now_year - 1) and (not eta.get('dvd') and not eta.get('theater') or eta.get('dvd') and eta.get('dvd') > (now - 2419200))) or
-                            (late and (media['info']['year'] < now_year - 1 or (eta.get('dvd', 0) > 0 or eta.get('theater')) and eta.get('dvd') < (now - 2419200)))):
+                    eta_date = eta.get(coming_soon)
+                    eta_3month_passed = eta_date < (now - 7862400)  # Release was more than 3 months ago
+
+                    if (not late and not eta_3month_passed) or \
+                            (late and eta_3month_passed):
 
                         add = True
 
                         # Check if it doesn't have any releases
                         if late:
                             media['releases'] = fireEvent('release.for_media', media['_id'], single = True)
-                            
+
                             for release in media.get('releases'):
                                 if release.get('status') in ['snatched', 'available', 'seeding', 'downloaded']:
                                     add = False

couchpotato/core/plugins/manage.py

@@ -165,7 +165,7 @@ class Manage(Plugin):
                                         already_used = used_files.get(release_file)
 
                                         if already_used:
-                                            release_id = release['_id'] if already_used.get('last_edit', 0) < release.get('last_edit', 0) else already_used['_id']
+                                            release_id = release['_id'] if already_used.get('last_edit', 0) > release.get('last_edit', 0) else already_used['_id']
                                             if release_id not in deleted_releases:
                                                 fireEvent('release.delete', release_id, single = True)
                                                 deleted_releases.append(release_id)
@@ -219,7 +219,7 @@ class Manage(Plugin):
 
                 # Add it to release and update the info
                 fireEvent('release.add', group = group, update_info = False)
-                fireEvent('movie.update_info', identifier = group['identifier'], on_complete = self.createAfterUpdate(folder, group['identifier']))
+                fireEvent('movie.update', identifier = group['identifier'], on_complete = self.createAfterUpdate(folder, group['identifier']))
 
         return addToLibrary
 

couchpotato/core/plugins/quality/main.py

@@ -1,3 +1,4 @@
+from math import fabs, ceil
 import traceback
 import re
 
@@ -6,7 +7,7 @@ from couchpotato import get_db
 from couchpotato.api import addApiView
 from couchpotato.core.event import addEvent, fireEvent
 from couchpotato.core.helpers.encoding import toUnicode, ss
-from couchpotato.core.helpers.variable import mergeDicts, getExt, tryInt, splitString
+from couchpotato.core.helpers.variable import mergeDicts, getExt, tryInt, splitString, tryFloat
 from couchpotato.core.logger import CPLog
 from couchpotato.core.plugins.base import Plugin
 from couchpotato.core.plugins.quality.index import QualityIndex
@@ -22,17 +23,17 @@ class QualityPlugin(Plugin):
     }
 
     qualities = [
-        {'identifier': 'bd50', 'hd': True, 'allow_3d': True, 'size': (20000, 60000), 'label': 'BR-Disk', 'alternative': ['bd25', ('br', 'disk')], 'allow': ['1080p'], 'ext':['iso', 'img'], 'tags': ['bdmv', 'certificate', ('complete', 'bluray'), 'avc', 'mvc']},
-        {'identifier': '1080p', 'hd': True, 'allow_3d': True, 'size': (4000, 20000), 'label': '1080p', 'width': 1920, 'height': 1080, 'alternative': [], 'allow': [], 'ext':['mkv', 'm2ts', 'ts'], 'tags': ['m2ts', 'x264', 'h264']},
-        {'identifier': '720p', 'hd': True, 'allow_3d': True, 'size': (3000, 10000), 'label': '720p', 'width': 1280, 'height': 720, 'alternative': [], 'allow': [], 'ext':['mkv', 'ts'], 'tags': ['x264', 'h264']},
-        {'identifier': 'brrip', 'hd': True, 'allow_3d': True, 'size': (700, 7000), 'label': 'BR-Rip', 'alternative': ['bdrip', ('br', 'rip')], 'allow': ['720p', '1080p'], 'ext':['mp4', 'avi'], 'tags': ['hdtv', 'hdrip', 'webdl', ('web', 'dl')]},
-        {'identifier': 'dvdr', 'size': (3000, 10000), 'label': 'DVD-R', 'alternative': ['br2dvd', ('dvd', 'r')], 'allow': [], 'ext':['iso', 'img', 'vob'], 'tags': ['pal', 'ntsc', 'video_ts', 'audio_ts', ('dvd', 'r'), 'dvd9']},
-        {'identifier': 'dvdrip', 'size': (600, 2400), 'label': 'DVD-Rip', 'width': 720, 'alternative': [('dvd', 'rip')], 'allow': [], 'ext':['avi'], 'tags': [('dvd', 'rip'), ('dvd', 'xvid'), ('dvd', 'divx')]},
-        {'identifier': 'scr', 'size': (600, 1600), 'label': 'Screener', 'alternative': ['screener', 'dvdscr', 'ppvrip', 'dvdscreener', 'hdscr'], 'allow': ['dvdr', 'dvdrip', '720p', '1080p'], 'ext':[], 'tags': ['webrip', ('web', 'rip')]},
-        {'identifier': 'r5', 'size': (600, 1000), 'label': 'R5', 'alternative': ['r6'], 'allow': ['dvdr', '720p'], 'ext':[]},
-        {'identifier': 'tc', 'size': (600, 1000), 'label': 'TeleCine', 'alternative': ['telecine'], 'allow': ['720p'], 'ext':[]},
-        {'identifier': 'ts', 'size': (600, 1000), 'label': 'TeleSync', 'alternative': ['telesync', 'hdts'], 'allow': ['720p'], 'ext':[]},
-        {'identifier': 'cam', 'size': (600, 1000), 'label': 'Cam', 'alternative': ['camrip', 'hdcam'], 'allow': ['720p'], 'ext':[]}
+        {'identifier': 'bd50', 'hd': True, 'allow_3d': True, 'size': (20000, 60000), 'median_size': 40000, 'label': 'BR-Disk', 'alternative': ['bd25', ('br', 'disk')], 'allow': ['1080p'], 'ext':['iso', 'img'], 'tags': ['bdmv', 'certificate', ('complete', 'bluray'), 'avc', 'mvc']},
+        {'identifier': '1080p', 'hd': True, 'allow_3d': True, 'size': (4000, 20000), 'median_size': 10000, 'label': '1080p', 'width': 1920, 'height': 1080, 'alternative': [], 'allow': [], 'ext':['mkv', 'm2ts', 'ts'], 'tags': ['m2ts', 'x264', 'h264']},
+        {'identifier': '720p', 'hd': True, 'allow_3d': True, 'size': (3000, 10000), 'median_size': 5500, 'label': '720p', 'width': 1280, 'height': 720, 'alternative': [], 'allow': [], 'ext':['mkv', 'ts'], 'tags': ['x264', 'h264']},
+        {'identifier': 'brrip', 'hd': True, 'allow_3d': True, 'size': (700, 7000), 'median_size': 2000, 'label': 'BR-Rip', 'alternative': ['bdrip', ('br', 'rip')], 'allow': ['720p', '1080p'], 'ext':['mp4', 'avi'], 'tags': ['hdtv', 'hdrip', 'webdl', ('web', 'dl')]},
+        {'identifier': 'dvdr', 'size': (3000, 10000), 'median_size': 4500, 'label': 'DVD-R', 'alternative': ['br2dvd', ('dvd', 'r')], 'allow': [], 'ext':['iso', 'img', 'vob'], 'tags': ['pal', 'ntsc', 'video_ts', 'audio_ts', ('dvd', 'r'), 'dvd9']},
+        {'identifier': 'dvdrip', 'size': (600, 2400), 'median_size': 1500, 'label': 'DVD-Rip', 'width': 720, 'alternative': [('dvd', 'rip')], 'allow': [], 'ext':['avi'], 'tags': [('dvd', 'rip'), ('dvd', 'xvid'), ('dvd', 'divx')]},
+        {'identifier': 'scr', 'size': (600, 1600), 'median_size': 700, 'label': 'Screener', 'alternative': ['screener', 'dvdscr', 'ppvrip', 'dvdscreener', 'hdscr'], 'allow': ['dvdr', 'dvdrip', '720p', '1080p'], 'ext':[], 'tags': ['webrip', ('web', 'rip')]},
+        {'identifier': 'r5', 'size': (600, 1000), 'median_size': 700, 'label': 'R5', 'alternative': ['r6'], 'allow': ['dvdr', '720p'], 'ext':[]},
+        {'identifier': 'tc', 'size': (600, 1000), 'median_size': 700, 'label': 'TeleCine', 'alternative': ['telecine'], 'allow': ['720p'], 'ext':[]},
+        {'identifier': 'ts', 'size': (600, 1000), 'median_size': 700, 'label': 'TeleSync', 'alternative': ['telesync', 'hdts'], 'allow': ['720p'], 'ext':[]},
+        {'identifier': 'cam', 'size': (600, 1000), 'median_size': 700, 'label': 'Cam', 'alternative': ['camrip', 'hdcam'], 'allow': ['720p'], 'ext':[]}
     ]
     pre_releases = ['cam', 'ts', 'tc', 'r5', 'scr']
     threed_tags = {
@@ -187,14 +188,15 @@ class QualityPlugin(Plugin):
 
         return False
 
-    def guess(self, files, extra = None, size = None):
+    def guess(self, files, extra = None, size = None, use_cache = True):
         if not extra: extra = {}
 
         # Create hash for cache
         cache_key = str([f.replace('.' + getExt(f), '') if len(getExt(f)) < 4 else f for f in files])
-        cached = self.getCache(cache_key)
-        if cached and len(extra) == 0:
-            return cached
+        if use_cache:
+            cached = self.getCache(cache_key)
+            if cached and len(extra) == 0:
+                return cached
 
         qualities = self.all()
 
@@ -206,6 +208,10 @@ class QualityPlugin(Plugin):
                 '3d': {}
             }
 
+        # Use metadata titles as extra check
+        if extra and extra.get('titles'):
+            files.extend(extra.get('titles'))
+
         for cur_file in files:
             words = re.split('\W+', cur_file.lower())
             name_year = fireEvent('scanner.name_year', cur_file, file_name = cur_file, single = True)
@@ -218,7 +224,7 @@ class QualityPlugin(Plugin):
                 contains_score = self.containsTagScore(quality, words, cur_file)
                 threedscore = self.contains3D(quality, threed_words, cur_file) if quality.get('allow_3d') else (0, None)
 
-                self.calcScore(score, quality, contains_score, threedscore)
+                self.calcScore(score, quality, contains_score, threedscore, penalty = contains_score)
 
         size_scores = []
         for quality in qualities:
@@ -230,11 +236,11 @@ class QualityPlugin(Plugin):
             if size_score > 0:
                 size_scores.append(quality)
 
-            self.calcScore(score, quality, size_score + loose_score, penalty = False)
+            self.calcScore(score, quality, size_score + loose_score)
 
         # Add additional size score if only 1 size validated
         if len(size_scores) == 1:
-            self.calcScore(score, size_scores[0], 10, penalty = False)
+            self.calcScore(score, size_scores[0], 8)
         del size_scores
 
         # Return nothing if all scores are <= 0
@@ -259,17 +265,17 @@ class QualityPlugin(Plugin):
 
     def containsTagScore(self, quality, words, cur_file = ''):
         cur_file = ss(cur_file)
-        score = 0
+        score = 0.0
 
         extension = words[-1]
         words = words[:-1]
 
         points = {
-            'identifier': 10,
-            'label': 10,
-            'alternative': 9,
-            'tags': 9,
-            'ext': 3,
+            'identifier': 20,
+            'label': 20,
+            'alternative': 20,
+            'tags': 11,
+            'ext': 5,
         }
 
         # Check alt and tags
@@ -285,7 +291,7 @@ class QualityPlugin(Plugin):
 
                 if isinstance(alt, (str, unicode)) and ss(alt.lower()) in words:
                     log.debug('Found %s via %s %s in %s', (quality['identifier'], tag_type, quality.get(tag_type), cur_file))
-                    score += points.get(tag_type) / 2
+                    score += points.get(tag_type)
 
             if list(set(qualities) & set(words)):
                 log.debug('Found %s via %s %s in %s', (quality['identifier'], tag_type, quality.get(tag_type), cur_file))
@@ -325,7 +331,7 @@ class QualityPlugin(Plugin):
             # Check width resolution, range 20
             if quality.get('width') and (quality.get('width') - 20) <= extra.get('resolution_width', 0) <= (quality.get('width') + 20):
                 log.debug('Found %s via resolution_width: %s == %s', (quality['identifier'], quality.get('width'), extra.get('resolution_width', 0)))
-                score += 5
+                score += 10
 
             # Check height resolution, range 20
             if quality.get('height') and (quality.get('height') - 20) <= extra.get('resolution_height', 0) <= (quality.get('height') + 20):
@@ -345,15 +351,28 @@ class QualityPlugin(Plugin):
 
         if size:
 
-            if tryInt(quality['size_min']) <= tryInt(size) <= tryInt(quality['size_max']):
-                log.debug('Found %s via release size: %s MB < %s MB < %s MB', (quality['identifier'], quality['size_min'], size, quality['size_max']))
-                score += 5
+            size = tryFloat(size)
+            size_min = tryFloat(quality['size_min'])
+            size_max = tryFloat(quality['size_max'])
+
+            if size_min <= size <= size_max:
+                log.debug('Found %s via release size: %s MB < %s MB < %s MB', (quality['identifier'], size_min, size, size_max))
+
+                proc_range = size_max - size_min
+                size_diff = size - size_min
+                size_proc = (size_diff / proc_range)
+
+                median_diff = quality['median_size'] - size_min
+                median_proc = (median_diff / proc_range)
+
+                max_points = 8
+                score += ceil(max_points - (fabs(size_proc - median_proc) * max_points))
             else:
                 score -= 5
 
         return score
 
-    def calcScore(self, score, quality, add_score, threedscore = (0, None), penalty = True):
+    def calcScore(self, score, quality, add_score, threedscore = (0, None), penalty = 0):
 
         score[quality['identifier']]['score'] += add_score
 
@@ -372,11 +391,11 @@ class QualityPlugin(Plugin):
 
         if penalty and add_score != 0:
             for allow in quality.get('allow', []):
-                score[allow]['score'] -= 40 if self.cached_order[allow] < self.cached_order[quality['identifier']] else 5
+                score[allow]['score'] -= ((penalty * 2) if self.cached_order[allow] < self.cached_order[quality['identifier']] else penalty) * 2
 
-            # Give panelty for all lower qualities
-            for q in self.qualities[self.order.index(quality.get('identifier'))+1:]:
-                if score.get(q.get('identifier')):
+            # Give panelty for all other qualities
+            for q in self.qualities:
+                if quality.get('identifier') != q.get('identifier') and score.get(q.get('identifier')):
                     score[q.get('identifier')]['score'] -= 1
 
     def isFinish(self, quality, profile, release_age = 0):
@@ -444,10 +463,12 @@ class QualityPlugin(Plugin):
             'Movie Monuments 2013 BrRip 1080p': {'size': 1800, 'quality': 'brrip'},
             'Movie Monuments 2013 BrRip 720p': {'size': 1300, 'quality': 'brrip'},
             'The.Movie.2014.3D.1080p.BluRay.AVC.DTS-HD.MA.5.1-GroupName': {'size': 30000, 'quality': 'bd50', 'is_3d': True},
-            '/home/namehou/Movie Monuments (2013)/Movie Monuments.mkv': {'size': 4500, 'quality': '1080p', 'is_3d': False},
-            '/home/namehou/Movie Monuments (2013)/Movie Monuments Full-OU.mkv': {'size': 4500, 'quality': '1080p', 'is_3d': True},
+            '/home/namehou/Movie Monuments (2012)/Movie Monuments.mkv': {'size': 5500, 'quality': '720p', 'is_3d': False},
+            '/home/namehou/Movie Monuments (2012)/Movie Monuments Full-OU.mkv': {'size': 5500, 'quality': '720p', 'is_3d': True},
+            '/home/namehou/Movie Monuments (2013)/Movie Monuments.mkv': {'size': 10000, 'quality': '1080p', 'is_3d': False},
+            '/home/namehou/Movie Monuments (2013)/Movie Monuments Full-OU.mkv': {'size': 10000, 'quality': '1080p', 'is_3d': True},
             '/volume1/Public/3D/Moviename/Moviename (2009).3D.SBS.ts': {'size': 7500, 'quality': '1080p', 'is_3d': True},
-            '/volume1/Public/Moviename/Moviename (2009).ts': {'size': 5500, 'quality': '1080p'},
+            '/volume1/Public/Moviename/Moviename (2009).ts': {'size': 7500, 'quality': '1080p'},
             '/movies/BluRay HDDVD H.264 MKV 720p EngSub/QuiQui le fou (criterion collection #123, 1915)/QuiQui le fou (1915) 720p x264 BluRay.mkv': {'size': 5500, 'quality': '720p'},
             'C:\\movies\QuiQui le fou (collection #123, 1915)\QuiQui le fou (1915) 720p x264 BluRay.mkv': {'size': 5500, 'quality': '720p'},
             'C:\\movies\QuiQui le fou (collection #123, 1915)\QuiQui le fou (1915) half-sbs 720p x264 BluRay.mkv': {'size': 5500, 'quality': '720p', 'is_3d': True},
@@ -456,12 +477,20 @@ class QualityPlugin(Plugin):
             'Movie Name (2014).mp4': {'size': 750, 'quality': 'brrip'},
             'Moviename.2014.720p.R6.WEB-DL.x264.AC3-xyz': {'size': 750, 'quality': 'r5'},
             'Movie name 2014 New Source 720p HDCAM x264 AC3 xyz': {'size': 750, 'quality': 'cam'},
-            'Movie.Name.2014.720p.HD.TS.AC3.x264': {'size': 750, 'quality': 'ts'}
+            'Movie.Name.2014.720p.HD.TS.AC3.x264': {'size': 750, 'quality': 'ts'},
+            # 'Movie.Name.2014.1080p.HDrip.x264.aac-ReleaseGroup': {'size': 7500, 'quality': 'brrip'},
+            'Movie.Name.2014.HDCam.Chinese.Subs-ReleaseGroup': {'size': 15000, 'quality': 'cam'},
+            'Movie Name 2014 HQ DVDRip X264 AC3 (bla)': {'size': 0, 'quality': 'dvdrip'},
+            'Movie Name1 (2012).mkv': {'size': 4500, 'quality': '720p'},
+            'Movie Name (2013).mkv': {'size': 8500, 'quality': '1080p'},
+            'Movie Name (2014).mkv': {'size': 4500, 'quality': '720p', 'extra': {'titles': ['Movie Name 2014 720p Bluray']}},
+            'Movie Name (2015).mkv': {'size': 500, 'quality': '1080p', 'extra': {'resolution_width': 1920}},
+            'Movie Name (2015).mp4': {'size': 6500, 'quality': 'brrip'},
         }
 
         correct = 0
         for name in tests:
-            test_quality = self.guess(files = [name], extra = tests[name].get('extra', None), size = tests[name].get('size', None)) or {}
+            test_quality = self.guess(files = [name], extra = tests[name].get('extra', None), size = tests[name].get('size', None), use_cache = False) or {}
             success = test_quality.get('identifier') == tests[name]['quality'] and test_quality.get('is_3d') == tests[name].get('is_3d', False)
             if not success:
                 log.error('%s failed check, thinks it\'s "%s" expecting "%s"', (name,

couchpotato/core/plugins/release/main.py

@@ -325,7 +325,7 @@ class Release(Plugin):
                 rls['download_info'] = download_result
                 db.update(rls)
 
-            log_movie = '%s (%s) in %s' % (getTitle(media), media['info']['year'], rls['quality'])
+            log_movie = '%s (%s) in %s' % (getTitle(media), media['info'].get('year'), rls['quality'])
             snatch_message = 'Snatched "%s": %s' % (data.get('name'), log_movie)
             log.info(snatch_message)
             fireEvent('%s.snatched' % data['type'], message = snatch_message, data = media)

couchpotato/core/plugins/renamer.py

@@ -247,7 +247,7 @@ class Renamer(Plugin):
                         'profile_id': None
                     }, search_after = False, status = 'done', single = True)
                 else:
-                    group['media'] = fireEvent('movie.update_info', media_id = group['media'].get('_id'), single = True)
+                    group['media'] = fireEvent('movie.update', media_id = group['media'].get('_id'), single = True)
 
                 if not group['media'] or not group['media'].get('_id'):
                     log.error('Could not rename, no library item to work with: %s', group_identifier)

couchpotato/core/plugins/scanner.py

@@ -11,7 +11,6 @@ from couchpotato.core.helpers.variable import getExt, getImdb, tryInt, \
     splitString, getIdentifier
 from couchpotato.core.logger import CPLog
 from couchpotato.core.plugins.base import Plugin
-from enzyme.exceptions import NoParserError, ParseError
 from guessit import guess_movie_info
 from subliminal.videos import Video
 import enzyme
@@ -457,6 +456,7 @@ class Scanner(Plugin):
                 meta = self.getMeta(cur_file)
 
                 try:
+                    data['titles'] = meta.get('titles', [])
                     data['video'] = meta.get('video', self.getCodec(cur_file, self.codecs['video']))
                     data['audio'] = meta.get('audio', self.getCodec(cur_file, self.codecs['audio']))
                     data['audio_channels'] = meta.get('audio_channels', 2.0)
@@ -527,16 +527,33 @@ class Scanner(Plugin):
             try: ac = self.audio_codec_map.get(p.audio[0].codec)
             except: pass
 
+            # Find title in video headers
+            titles = []
+
+            try:
+                if p.title and self.findYear(p.title):
+                    titles.append(ss(p.title))
+            except:
+                log.error('Failed getting title from meta: %s', traceback.format_exc())
+
+            for video in p.video:
+                try:
+                    if video.title and self.findYear(video.title):
+                        titles.append(ss(video.title))
+                except:
+                    log.error('Failed getting title from meta: %s', traceback.format_exc())
+
             return {
+                'titles': list(set(titles)),
                 'video': vc,
                 'audio': ac,
                 'resolution_width': tryInt(p.video[0].width),
                 'resolution_height': tryInt(p.video[0].height),
                 'audio_channels': p.audio[0].channels,
             }
-        except ParseError:
+        except enzyme.exceptions.ParseError:
             log.debug('Failed to parse meta for %s', filename)
-        except NoParserError:
+        except enzyme.exceptions.NoParserError:
             log.debug('No parser found for %s', filename)
         except:
             log.debug('Failed parsing %s', filename)

couchpotato/core/plugins/score/scores.py

@@ -33,33 +33,43 @@ name_scores = [
 def nameScore(name, year, preferred_words):
     """ Calculate score for words in the NZB name """
 
-    score = 0
-    name = name.lower()
+    try:
+        score = 0
+        name = name.lower()
 
-    # give points for the cool stuff
-    for value in name_scores:
-        v = value.split(':')
-        add = int(v.pop())
-        if v.pop() in name:
-            score += add
+        # give points for the cool stuff
+        for value in name_scores:
+            v = value.split(':')
+            add = int(v.pop())
+            if v.pop() in name:
+                score += add
 
-    # points if the year is correct
-    if str(year) in name:
-        score += 5
+        # points if the year is correct
+        if str(year) in name:
+            score += 5
 
-    # Contains preferred word
-    nzb_words = re.split('\W+', simplifyString(name))
-    score += 100 * len(list(set(nzb_words) & set(preferred_words)))
+        # Contains preferred word
+        nzb_words = re.split('\W+', simplifyString(name))
+        score += 100 * len(list(set(nzb_words) & set(preferred_words)))
 
-    return score
+        return score
+    except:
+        log.error('Failed doing nameScore: %s', traceback.format_exc())
+
+    return 0
 
 
 def nameRatioScore(nzb_name, movie_name):
-    nzb_words = re.split('\W+', fireEvent('scanner.create_file_identifier', nzb_name, single = True))
-    movie_words = re.split('\W+', simplifyString(movie_name))
+    try:
+        nzb_words = re.split('\W+', fireEvent('scanner.create_file_identifier', nzb_name, single = True))
+        movie_words = re.split('\W+', simplifyString(movie_name))
 
-    left_over = set(nzb_words) - set(movie_words)
-    return 10 - len(left_over)
+        left_over = set(nzb_words) - set(movie_words)
+        return 10 - len(left_over)
+    except:
+        log.error('Failed doing nameRatioScore: %s', traceback.format_exc())
+
+    return 0
 
 
 def namePositionScore(nzb_name, movie_name):
@@ -134,38 +144,53 @@ def providerScore(provider):
 
 def duplicateScore(nzb_name, movie_name):
 
-    nzb_words = re.split('\W+', simplifyString(nzb_name))
-    movie_words = re.split('\W+', simplifyString(movie_name))
+    try:
+        nzb_words = re.split('\W+', simplifyString(nzb_name))
+        movie_words = re.split('\W+', simplifyString(movie_name))
 
-    # minus for duplicates
-    duplicates = [x for i, x in enumerate(nzb_words) if nzb_words[i:].count(x) > 1]
+        # minus for duplicates
+        duplicates = [x for i, x in enumerate(nzb_words) if nzb_words[i:].count(x) > 1]
 
-    return len(list(set(duplicates) - set(movie_words))) * -4
+        return len(list(set(duplicates) - set(movie_words))) * -4
+    except:
+        log.error('Failed doing duplicateScore: %s', traceback.format_exc())
+
+    return 0
 
 
 def partialIgnoredScore(nzb_name, movie_name, ignored_words):
 
-    nzb_name = nzb_name.lower()
-    movie_name = movie_name.lower()
+    try:
+        nzb_name = nzb_name.lower()
+        movie_name = movie_name.lower()
 
-    score = 0
-    for ignored_word in ignored_words:
-        if ignored_word in nzb_name and ignored_word not in movie_name:
-            score -= 5
+        score = 0
+        for ignored_word in ignored_words:
+            if ignored_word in nzb_name and ignored_word not in movie_name:
+                score -= 5
 
-    return score
+        return score
+    except:
+        log.error('Failed doing partialIgnoredScore: %s', traceback.format_exc())
+
+    return 0
 
 
 def halfMultipartScore(nzb_name):
 
-    wrong_found = 0
-    for nr in [1, 2, 3, 4, 5, 'i', 'ii', 'iii', 'iv', 'v', 'a', 'b', 'c', 'd', 'e']:
-        for wrong in ['cd', 'part', 'dis', 'disc', 'dvd']:
-            if '%s%s' % (wrong, nr) in nzb_name.lower():
-                wrong_found += 1
+    try:
+        wrong_found = 0
+        for nr in [1, 2, 3, 4, 5, 'i', 'ii', 'iii', 'iv', 'v', 'a', 'b', 'c', 'd', 'e']:
+            for wrong in ['cd', 'part', 'dis', 'disc', 'dvd']:
+                if '%s%s' % (wrong, nr) in nzb_name.lower():
+                    wrong_found += 1
 
-    if wrong_found == 1:
-        return -30
+        if wrong_found == 1:
+            return -30
+
+        return 0
+    except:
+        log.error('Failed doing halfMultipartScore: %s', traceback.format_exc())
 
     return 0
 

libs/tornado/__init__.py

@@ -25,5 +25,5 @@ from __future__ import absolute_import, division, print_function, with_statement
 # is zero for an official release, positive for a development branch,
 # or negative for a release candidate or beta (after the base version
 # number has been incremented)
-version = "3.3.dev1"
-version_info = (3, 3, 0, -100)
+version = "4.0.1"
+version_info = (4, 0, 1, -100)

libs/tornado/auth.py

@@ -51,7 +51,7 @@ Example usage for Google OpenID::
                     response_type='code',
                     extra_params={'approval_prompt': 'auto'})
 
-.. versionchanged:: 3.3
+.. versionchanged:: 4.0
    All of the callback interfaces in this module are now guaranteed
    to run their callback with an argument of ``None`` on error.
    Previously some functions would do this while others would simply
@@ -883,9 +883,10 @@ class FriendFeedMixin(OAuthMixin):
 class GoogleMixin(OpenIdMixin, OAuthMixin):
     """Google Open ID / OAuth authentication.
 
-    *Deprecated:* New applications should use `GoogleOAuth2Mixin`
-    below instead of this class. As of May 19, 2014, Google has stopped
-    supporting registration-free authentication.
+    .. deprecated:: 4.0
+       New applications should use `GoogleOAuth2Mixin`
+       below instead of this class. As of May 19, 2014, Google has stopped
+       supporting registration-free authentication.
 
     No application registration is necessary to use Google for
     authentication or to access Google resources on behalf of a user.
@@ -1053,9 +1054,10 @@ class GoogleOAuth2Mixin(OAuth2Mixin):
 class FacebookMixin(object):
     """Facebook Connect authentication.
 
-    *Deprecated:* New applications should use `FacebookGraphMixin`
-    below instead of this class.  This class does not support the
-    Future-based interface seen on other classes in this module.
+    .. deprecated:: 1.1
+       New applications should use `FacebookGraphMixin`
+       below instead of this class.  This class does not support the
+       Future-based interface seen on other classes in this module.
 
     To authenticate with Facebook, register your application with
     Facebook at http://www.facebook.com/developers/apps.php. Then

libs/tornado/concurrent.py

@@ -60,7 +60,7 @@ class Future(object):
     This functionality was previously available in a separate class
     ``TracebackFuture``, which is now a deprecated alias for this class.
 
-    .. versionchanged:: 3.3
+    .. versionchanged:: 4.0
        `tornado.concurrent.Future` is always a thread-unsafe ``Future``
        with support for the ``exc_info`` methods.  Previously it would
        be an alias for the thread-safe `concurrent.futures.Future`
@@ -152,7 +152,7 @@ class Future(object):
     def exc_info(self):
         """Returns a tuple in the same format as `sys.exc_info` or None.
 
-        .. versionadded:: 3.3
+        .. versionadded:: 4.0
         """
         return self._exc_info
 
@@ -161,7 +161,7 @@ class Future(object):
 
         Preserves tracebacks on Python 2.
 
-        .. versionadded:: 3.3
+        .. versionadded:: 4.0
         """
         self._exc_info = exc_info
         self.set_exception(exc_info[1])

libs/tornado/curl_httpclient.py

@@ -51,18 +51,6 @@ class CurlAsyncHTTPClient(AsyncHTTPClient):
         self._fds = {}
         self._timeout = None
 
-        try:
-            self._socket_action = self._multi.socket_action
-        except AttributeError:
-            # socket_action is found in pycurl since 7.18.2 (it's been
-            # in libcurl longer than that but wasn't accessible to
-            # python).
-            gen_log.warning("socket_action method missing from pycurl; "
-                            "falling back to socket_all. Upgrading "
-                            "libcurl and pycurl will improve performance")
-            self._socket_action = \
-                lambda fd, action: self._multi.socket_all()
-
         # libcurl has bugs that sometimes cause it to not report all
         # relevant file descriptors and timeouts to TIMERFUNCTION/
         # SOCKETFUNCTION.  Mitigate the effects of such bugs by
@@ -87,7 +75,6 @@ class CurlAsyncHTTPClient(AsyncHTTPClient):
         for curl in self._curls:
             curl.close()
         self._multi.close()
-        self._closed = True
         super(CurlAsyncHTTPClient, self).close()
 
     def fetch_impl(self, request, callback):
@@ -143,7 +130,7 @@ class CurlAsyncHTTPClient(AsyncHTTPClient):
             action |= pycurl.CSELECT_OUT
         while True:
             try:
-                ret, num_handles = self._socket_action(fd, action)
+                ret, num_handles = self._multi.socket_action(fd, action)
             except pycurl.error as e:
                 ret = e.args[0]
             if ret != pycurl.E_CALL_MULTI_PERFORM:
@@ -156,7 +143,7 @@ class CurlAsyncHTTPClient(AsyncHTTPClient):
             self._timeout = None
             while True:
                 try:
-                    ret, num_handles = self._socket_action(
+                    ret, num_handles = self._multi.socket_action(
                         pycurl.SOCKET_TIMEOUT, 0)
                 except pycurl.error as e:
                     ret = e.args[0]
@@ -224,11 +211,6 @@ class CurlAsyncHTTPClient(AsyncHTTPClient):
                         "callback": callback,
                         "curl_start_time": time.time(),
                     }
-                    # Disable IPv6 to mitigate the effects of this bug
-                    # on curl versions <= 7.21.0
-                    # http://sourceforge.net/tracker/?func=detail&aid=3017819&group_id=976&atid=100976
-                    if pycurl.version_info()[2] <= 0x71500:  # 7.21.0
-                        curl.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_V4)
                     _curl_setup_request(curl, request, curl.info["buffer"],
                                         curl.info["headers"])
                     self._multi.add_handle(curl)
@@ -350,7 +332,7 @@ def _curl_setup_request(curl, request, buffer, headers):
         curl.setopt(pycurl.USERAGENT, "Mozilla/5.0 (compatible; pycurl)")
     if request.network_interface:
         curl.setopt(pycurl.INTERFACE, request.network_interface)
-    if request.use_gzip:
+    if request.decompress_response:
         curl.setopt(pycurl.ENCODING, "gzip,deflate")
     else:
         curl.setopt(pycurl.ENCODING, "none")
@@ -384,7 +366,6 @@ def _curl_setup_request(curl, request, buffer, headers):
     if request.allow_ipv6 is False:
         # Curl behaves reasonably when DNS resolution gives an ipv6 address
         # that we can't reach, so allow ipv6 unless the user asks to disable.
-        # (but see version check in _process_queue above)
         curl.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_V4)
     else:
         curl.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_WHATEVER)
@@ -474,7 +455,7 @@ def _curl_header_callback(headers, header_line):
         try:
             (__, __, reason) = httputil.parse_response_start_line(header_line)
             header_line = "X-Http-Reason: %s" % reason
-        except httputil.HTTPInputException:
+        except httputil.HTTPInputError:
             return
     if not header_line:
         return

libs/tornado/gen.py

@@ -29,16 +29,7 @@ could be written with ``gen`` as::
 Most asynchronous functions in Tornado return a `.Future`;
 yielding this object returns its `~.Future.result`.
 
-For functions that do not return ``Futures``, `Task` works with any
-function that takes a ``callback`` keyword argument (most Tornado functions
-can be used in either style, although the ``Future`` style is preferred
-since it is both shorter and provides better exception handling)::
-
-    @gen.coroutine
-    def get(self):
-        yield gen.Task(AsyncHTTPClient().fetch, "http://example.com")
-
-You can also yield a list or dict of ``Futures`` and/or ``Tasks``, which will be
+You can also yield a list or dict of ``Futures``, which will be
 started at the same time and run in parallel; a list or dict of results will
 be returned when they are all finished::
 
@@ -54,30 +45,6 @@ be returned when they are all finished::
 
 .. versionchanged:: 3.2
    Dict support added.
-
-For more complicated interfaces, `Task` can be split into two parts:
-`Callback` and `Wait`::
-
-    class GenAsyncHandler2(RequestHandler):
-        @gen.coroutine
-        def get(self):
-            http_client = AsyncHTTPClient()
-            http_client.fetch("http://example.com",
-                              callback=(yield gen.Callback("key")))
-            response = yield gen.Wait("key")
-            do_something_with_response(response)
-            self.render("template.html")
-
-The ``key`` argument to `Callback` and `Wait` allows for multiple
-asynchronous operations to be started at different times and proceed
-in parallel: yield several callbacks with different keys, then wait
-for them once all the async operations have started.
-
-The result of a `Wait` or `Task` yield expression depends on how the callback
-was run.  If it was called with no arguments, the result is ``None``.  If
-it was called with one argument, the result is that argument.  If it was
-called with more than one argument or any keyword arguments, the result
-is an `Arguments` object, which is a named tuple ``(args, kwargs)``.
 """
 from __future__ import absolute_import, division, print_function, with_statement
 
@@ -252,8 +219,8 @@ class Return(Exception):
 class YieldPoint(object):
     """Base class for objects that may be yielded from the generator.
 
-    Applications do not normally need to use this class, but it may be
-    subclassed to provide additional yielding behavior.
+    .. deprecated:: 4.0
+       Use `Futures <.Future>` instead.
     """
     def start(self, runner):
         """Called by the runner after the generator has yielded.
@@ -289,6 +256,9 @@ class Callback(YieldPoint):
 
     The callback may be called with zero or one arguments; if an argument
     is given it will be returned by `Wait`.
+
+    .. deprecated:: 4.0
+       Use `Futures <.Future>` instead.
     """
     def __init__(self, key):
         self.key = key
@@ -305,7 +275,11 @@ class Callback(YieldPoint):
 
 
 class Wait(YieldPoint):
-    """Returns the argument passed to the result of a previous `Callback`."""
+    """Returns the argument passed to the result of a previous `Callback`.
+
+    .. deprecated:: 4.0
+       Use `Futures <.Future>` instead.
+    """
     def __init__(self, key):
         self.key = key
 
@@ -326,6 +300,9 @@ class WaitAll(YieldPoint):
     a list of results in the same order.
 
     `WaitAll` is equivalent to yielding a list of `Wait` objects.
+
+    .. deprecated:: 4.0
+       Use `Futures <.Future>` instead.
     """
     def __init__(self, keys):
         self.keys = keys
@@ -341,21 +318,13 @@ class WaitAll(YieldPoint):
 
 
 def Task(func, *args, **kwargs):
-    """Runs a single asynchronous operation.
+    """Adapts a callback-based asynchronous function for use in coroutines.
 
     Takes a function (and optional additional arguments) and runs it with
     those arguments plus a ``callback`` keyword argument.  The argument passed
     to the callback is returned as the result of the yield expression.
 
-    A `Task` is equivalent to a `Callback`/`Wait` pair (with a unique
-    key generated automatically)::
-
-        result = yield gen.Task(func, args)
-
-        func(args, callback=(yield gen.Callback(key)))
-        result = yield gen.Wait(key)
-
-    .. versionchanged:: 3.3
+    .. versionchanged:: 4.0
        ``gen.Task`` is now a function that returns a `.Future`, instead of
        a subclass of `YieldPoint`.  It still behaves the same way when
        yielded.
@@ -464,7 +433,7 @@ def multi_future(children):
     This function is faster than the `Multi` `YieldPoint` because it does not
     require the creation of a stack context.
 
-    .. versionadded:: 3.3
+    .. versionadded:: 4.0
     """
     if isinstance(children, dict):
         keys = list(children.keys())
@@ -520,7 +489,7 @@ def with_timeout(timeout, future, io_loop=None):
 
     Currently only supports Futures, not other `YieldPoint` classes.
 
-    .. versionadded:: 3.3
+    .. versionadded:: 4.0
     """
     # TODO: allow yield points in addition to futures?
     # Tricky to do with stack_context semantics.
@@ -564,7 +533,7 @@ coroutines that are likely to yield Futures that are ready instantly.
 
 Usage: ``yield gen.moment``
 
-.. versionadded:: 3.3
+.. versionadded:: 4.0
 """
 moment.set_result(None)
 

libs/tornado/http1connection.py

@@ -16,11 +16,13 @@
 
 """Client and server implementations of HTTP/1.x.
 
-.. versionadded:: 3.3
+.. versionadded:: 4.0
 """
 
 from __future__ import absolute_import, division, print_function, with_statement
 
+import re
+
 from tornado.concurrent import Future
 from tornado.escape import native_str, utf8
 from tornado import gen
@@ -56,7 +58,7 @@ class HTTP1ConnectionParameters(object):
     """
     def __init__(self, no_keep_alive=False, chunk_size=None,
                  max_header_size=None, header_timeout=None, max_body_size=None,
-                 body_timeout=None, use_gzip=False):
+                 body_timeout=None, decompress=False):
         """
         :arg bool no_keep_alive: If true, always close the connection after
             one request.
@@ -65,7 +67,8 @@ class HTTP1ConnectionParameters(object):
         :arg float header_timeout: how long to wait for all headers (seconds)
         :arg int max_body_size: maximum amount of data for body
         :arg float body_timeout: how long to wait while reading body (seconds)
-        :arg bool use_gzip: if true, decode incoming ``Content-Encoding: gzip``
+        :arg bool decompress: if true, decode incoming
+            ``Content-Encoding: gzip``
         """
         self.no_keep_alive = no_keep_alive
         self.chunk_size = chunk_size or 65536
@@ -73,7 +76,7 @@ class HTTP1ConnectionParameters(object):
         self.header_timeout = header_timeout
         self.max_body_size = max_body_size
         self.body_timeout = body_timeout
-        self.use_gzip = use_gzip
+        self.decompress = decompress
 
 
 class HTTP1Connection(httputil.HTTPConnection):
@@ -141,7 +144,7 @@ class HTTP1Connection(httputil.HTTPConnection):
         Returns a `.Future` that resolves to None after the full response has
         been read.
         """
-        if self.params.use_gzip:
+        if self.params.decompress:
             delegate = _GzipMessageDelegate(delegate, self.params.chunk_size)
         return self._read_message(delegate)
 
@@ -190,8 +193,17 @@ class HTTP1Connection(httputil.HTTPConnection):
                     skip_body = True
                 code = start_line.code
                 if code == 304:
+                    # 304 responses may include the content-length header
+                    # but do not actually have a body.
+                    # http://tools.ietf.org/html/rfc7230#section-3.3
                     skip_body = True
                 if code >= 100 and code < 200:
+                    # 1xx responses should never indicate the presence of
+                    # a body.
+                    if ('Content-Length' in headers or
+                        'Transfer-Encoding' in headers):
+                        raise httputil.HTTPInputError(
+                            "Response code %d cannot have body" % code)
                     # TODO: client delegates will get headers_received twice
                     # in the case of a 100-continue.  Document or change?
                     yield self._read_message(delegate)
@@ -200,7 +212,8 @@ class HTTP1Connection(httputil.HTTPConnection):
                         not self._write_finished):
                     self.stream.write(b"HTTP/1.1 100 (Continue)\r\n\r\n")
             if not skip_body:
-                body_future = self._read_body(headers, delegate)
+                body_future = self._read_body(
+                    start_line.code if self.is_client else 0, headers, delegate)
                 if body_future is not None:
                     if self._body_timeout is None:
                         yield body_future
@@ -231,7 +244,7 @@ class HTTP1Connection(httputil.HTTPConnection):
                 self.close()
             if self.stream is None:
                 raise gen.Return(False)
-        except httputil.HTTPInputException as e:
+        except httputil.HTTPInputError as e:
             gen_log.info("Malformed HTTP message from %s: %s",
                          self.context, e)
             self.close()
@@ -258,7 +271,7 @@ class HTTP1Connection(httputil.HTTPConnection):
     def set_close_callback(self, callback):
         """Sets a callback that will be run when the connection is closed.
 
-        .. deprecated:: 3.3
+        .. deprecated:: 4.0
             Use `.HTTPMessageDelegate.on_connection_close` instead.
         """
         self._close_callback = stack_context.wrap(callback)
@@ -377,7 +390,7 @@ class HTTP1Connection(httputil.HTTPConnection):
             if self._expected_content_remaining < 0:
                 # Close the stream now to stop further framing errors.
                 self.stream.close()
-                raise httputil.HTTPOutputException(
+                raise httputil.HTTPOutputError(
                     "Tried to write more data than Content-Length")
         if self._chunking_output and chunk:
             # Don't write out empty chunks because that means END-OF-STREAM
@@ -412,7 +425,7 @@ class HTTP1Connection(httputil.HTTPConnection):
                 self._expected_content_remaining != 0 and
                 not self.stream.closed()):
             self.stream.close()
-            raise httputil.HTTPOutputException(
+            raise httputil.HTTPOutputError(
                 "Tried to write %d bytes less than Content-Length" %
                 self._expected_content_remaining)
         if self._chunking_output:
@@ -477,16 +490,40 @@ class HTTP1Connection(httputil.HTTPConnection):
             headers = httputil.HTTPHeaders.parse(data[eol:])
         except ValueError:
             # probably form split() if there was no ':' in the line
-            raise httputil.HTTPInputException("Malformed HTTP headers: %r" %
-                                              data[eol:100])
+            raise httputil.HTTPInputError("Malformed HTTP headers: %r" %
+                                          data[eol:100])
         return start_line, headers
 
-    def _read_body(self, headers, delegate):
-        content_length = headers.get("Content-Length")
-        if content_length:
-            content_length = int(content_length)
+    def _read_body(self, code, headers, delegate):
+        if "Content-Length" in headers:
+            if "," in headers["Content-Length"]:
+                # Proxies sometimes cause Content-Length headers to get
+                # duplicated.  If all the values are identical then we can
+                # use them but if they differ it's an error.
+                pieces = re.split(r',\s*', headers["Content-Length"])
+                if any(i != pieces[0] for i in pieces):
+                    raise httputil.HTTPInputError(
+                        "Multiple unequal Content-Lengths: %r" %
+                        headers["Content-Length"])
+                headers["Content-Length"] = pieces[0]
+            content_length = int(headers["Content-Length"])
+
             if content_length > self._max_body_size:
-                raise httputil.HTTPInputException("Content-Length too long")
+                raise httputil.HTTPInputError("Content-Length too long")
+        else:
+            content_length = None
+
+        if code == 204:
+            # This response code is not allowed to have a non-empty body,
+            # and has an implicit length of zero instead of read-until-close.
+            # http://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.3
+            if ("Transfer-Encoding" in headers or
+                    content_length not in (None, 0)):
+                raise httputil.HTTPInputError(
+                    "Response with code %d should not have body" % code)
+            content_length = 0
+
+        if content_length is not None:
             return self._read_fixed_body(content_length, delegate)
         if headers.get("Transfer-Encoding") == "chunked":
             return self._read_chunked_body(delegate)
@@ -515,7 +552,7 @@ class HTTP1Connection(httputil.HTTPConnection):
                 return
             total_size += chunk_len
             if total_size > self._max_body_size:
-                raise httputil.HTTPInputException("chunked body too large")
+                raise httputil.HTTPInputError("chunked body too large")
             bytes_to_read = chunk_len
             while bytes_to_read:
                 chunk = yield self.stream.read_bytes(
@@ -581,6 +618,9 @@ class _GzipMessageDelegate(httputil.HTTPMessageDelegate):
                 self._delegate.data_received(tail)
         return self._delegate.finish()
 
+    def on_connection_close(self):
+        return self._delegate.on_connection_close()
+
 
 class HTTP1ServerConnection(object):
     """An HTTP/1.x server."""

libs/tornado/httpclient.py

@@ -22,14 +22,20 @@ to switch to ``curl_httpclient`` for reasons such as the following:
 
 * ``curl_httpclient`` was the default prior to Tornado 2.0.
 
-Note that if you are using ``curl_httpclient``, it is highly recommended that
-you use a recent version of ``libcurl`` and ``pycurl``.  Currently the minimum
-supported version is 7.18.2, and the recommended version is 7.21.1 or newer.
-It is highly recommended that your ``libcurl`` installation is built with
-asynchronous DNS resolver (threaded or c-ares), otherwise you may encounter
-various problems with request timeouts (for more information, see
+Note that if you are using ``curl_httpclient``, it is highly
+recommended that you use a recent version of ``libcurl`` and
+``pycurl``.  Currently the minimum supported version of libcurl is
+7.21.1, and the minimum version of pycurl is 7.18.2.  It is highly
+recommended that your ``libcurl`` installation is built with
+asynchronous DNS resolver (threaded or c-ares), otherwise you may
+encounter various problems with request timeouts (for more
+information, see
 http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTCONNECTTIMEOUTMS
 and comments in curl_httpclient.py).
+
+To select ``curl_httpclient``, call `AsyncHTTPClient.configure` at startup::
+
+    AsyncHTTPClient.configure("tornado.curl_httpclient.CurlAsyncHTTPClient")
 """
 
 from __future__ import absolute_import, division, print_function, with_statement
@@ -110,10 +116,21 @@ class AsyncHTTPClient(Configurable):
     actually creates an instance of an implementation-specific
     subclass, and instances are reused as a kind of pseudo-singleton
     (one per `.IOLoop`).  The keyword argument ``force_instance=True``
-    can be used to suppress this singleton behavior.  Constructor
-    arguments other than ``io_loop`` and ``force_instance`` are
-    deprecated.  The implementation subclass as well as arguments to
-    its constructor can be set with the static method `configure()`
+    can be used to suppress this singleton behavior.  Unless
+    ``force_instance=True`` is used, no arguments other than
+    ``io_loop`` should be passed to the `AsyncHTTPClient` constructor.
+    The implementation subclass as well as arguments to its
+    constructor can be set with the static method `configure()`
+
+    All `AsyncHTTPClient` implementations support a ``defaults``
+    keyword argument, which can be used to set default values for
+    `HTTPRequest` attributes.  For example::
+
+        AsyncHTTPClient.configure(
+            None, defaults=dict(user_agent="MyUserAgent"))
+        # or with force_instance:
+        client = AsyncHTTPClient(force_instance=True,
+            defaults=dict(user_agent="MyUserAgent"))
     """
     @classmethod
     def configurable_base(cls):
@@ -133,12 +150,21 @@ class AsyncHTTPClient(Configurable):
 
     def __new__(cls, io_loop=None, force_instance=False, **kwargs):
         io_loop = io_loop or IOLoop.current()
-        if io_loop in cls._async_clients() and not force_instance:
-            return cls._async_clients()[io_loop]
+        if force_instance:
+            instance_cache = None
+        else:
+            instance_cache = cls._async_clients()
+        if instance_cache is not None and io_loop in instance_cache:
+            return instance_cache[io_loop]
         instance = super(AsyncHTTPClient, cls).__new__(cls, io_loop=io_loop,
                                                        **kwargs)
-        if not force_instance:
-            cls._async_clients()[io_loop] = instance
+        # Make sure the instance knows which cache to remove itself from.
+        # It can't simply call _async_clients() because we may be in
+        # __new__(AsyncHTTPClient) but instance.__class__ may be
+        # SimpleAsyncHTTPClient.
+        instance._instance_cache = instance_cache
+        if instance_cache is not None:
+            instance_cache[instance.io_loop] = instance
         return instance
 
     def initialize(self, io_loop, defaults=None):
@@ -146,6 +172,7 @@ class AsyncHTTPClient(Configurable):
         self.defaults = dict(HTTPRequest._DEFAULTS)
         if defaults is not None:
             self.defaults.update(defaults)
+        self._closed = False
 
     def close(self):
         """Destroys this HTTP client, freeing any file descriptors used.
@@ -160,8 +187,13 @@ class AsyncHTTPClient(Configurable):
         ``close()``.
 
         """
-        if self._async_clients().get(self.io_loop) is self:
-            del self._async_clients()[self.io_loop]
+        if self._closed:
+            return
+        self._closed = True
+        if self._instance_cache is not None:
+            if self._instance_cache.get(self.io_loop) is not self:
+                raise RuntimeError("inconsistent AsyncHTTPClient cache")
+            del self._instance_cache[self.io_loop]
 
     def fetch(self, request, callback=None, **kwargs):
         """Executes a request, asynchronously returning an `HTTPResponse`.
@@ -179,6 +211,8 @@ class AsyncHTTPClient(Configurable):
         Instead, you must check the response's ``error`` attribute or
         call its `~HTTPResponse.rethrow` method.
         """
+        if self._closed:
+            raise RuntimeError("fetch() called on closed AsyncHTTPClient")
         if not isinstance(request, HTTPRequest):
             request = HTTPRequest(url=request, **kwargs)
         # We may modify this (to add Host, Accept-Encoding, etc),
@@ -248,7 +282,7 @@ class HTTPRequest(object):
         request_timeout=20.0,
         follow_redirects=True,
         max_redirects=5,
-        use_gzip=True,
+        decompress_response=True,
         proxy_password='',
         allow_nonstandard_methods=False,
         validate_cert=True)
@@ -265,7 +299,7 @@ class HTTPRequest(object):
                  validate_cert=None, ca_certs=None,
                  allow_ipv6=None,
                  client_key=None, client_cert=None, body_producer=None,
-                 expect_100_continue=False):
+                 expect_100_continue=False, decompress_response=None):
         r"""All parameters except ``url`` are optional.
 
         :arg string url: URL to fetch
@@ -284,7 +318,7 @@ class HTTPRequest(object):
            ``curl_httpclient``.  When using ``body_producer`` it is recommended
            to pass a ``Content-Length`` in the headers as otherwise chunked
            encoding will be used, and many servers do not support chunked
-           encoding on requests.  New in Tornado 3.3
+           encoding on requests.  New in Tornado 4.0
         :arg string auth_username: Username for HTTP authentication
         :arg string auth_password: Password for HTTP authentication
         :arg string auth_mode: Authentication mode; default is "basic".
@@ -299,7 +333,11 @@ class HTTPRequest(object):
            or return the 3xx response?
         :arg int max_redirects: Limit for ``follow_redirects``
         :arg string user_agent: String to send as ``User-Agent`` header
-        :arg bool use_gzip: Request gzip encoding from the server
+        :arg bool decompress_response: Request a compressed response from
+           the server and decompress it after downloading.  Default is True.
+           New in Tornado 4.0.
+        :arg bool use_gzip: Deprecated alias for ``decompress_response``
+           since Tornado 4.0.
         :arg string network_interface: Network interface to use for request.
            ``curl_httpclient`` only; see note below.
         :arg callable streaming_callback: If set, ``streaming_callback`` will
@@ -342,7 +380,6 @@ class HTTPRequest(object):
            before sending the request body.  Only supported with
            simple_httpclient.
 
-
         .. note::
 
             When using ``curl_httpclient`` certain options may be
@@ -358,7 +395,7 @@ class HTTPRequest(object):
         .. versionadded:: 3.1
            The ``auth_mode`` argument.
 
-        .. versionadded:: 3.3
+        .. versionadded:: 4.0
            The ``body_producer`` and ``expect_100_continue`` arguments.
         """
         # Note that some of these attributes go through property setters
@@ -383,7 +420,10 @@ class HTTPRequest(object):
         self.follow_redirects = follow_redirects
         self.max_redirects = max_redirects
         self.user_agent = user_agent
-        self.use_gzip = use_gzip
+        if decompress_response is not None:
+            self.decompress_response = decompress_response
+        else:
+            self.decompress_response = use_gzip
         self.network_interface = network_interface
         self.streaming_callback = streaming_callback
         self.header_callback = header_callback

libs/tornado/httpserver.py

@@ -20,7 +20,7 @@ Typical applications have little direct interaction with the `HTTPServer`
 class except to start a server at the beginning of the process
 (and even that is often done indirectly via `tornado.web.Application.listen`).
 
-.. versionchanged:: 3.3
+.. versionchanged:: 4.0
 
    The ``HTTPRequest`` class that used to live in this module has been moved
    to `tornado.httputil.HTTPServerRequest`.  The old name remains as an alias.
@@ -128,14 +128,15 @@ class HTTPServer(TCPServer, httputil.HTTPServerConnectionDelegate):
        servers if you want to create your listening sockets in some
        way other than `tornado.netutil.bind_sockets`.
 
-    .. versionchanged:: 3.3
-       Added ``gzip``, ``chunk_size``, ``max_header_size``,
+    .. versionchanged:: 4.0
+       Added ``decompress_request``, ``chunk_size``, ``max_header_size``,
        ``idle_connection_timeout``, ``body_timeout``, ``max_body_size``
        arguments.  Added support for `.HTTPServerConnectionDelegate`
        instances as ``request_callback``.
     """
     def __init__(self, request_callback, no_keep_alive=False, io_loop=None,
-                 xheaders=False, ssl_options=None, protocol=None, gzip=False,
+                 xheaders=False, ssl_options=None, protocol=None,
+                 decompress_request=False,
                  chunk_size=None, max_header_size=None,
                  idle_connection_timeout=None, body_timeout=None,
                  max_body_size=None, max_buffer_size=None):
@@ -144,7 +145,7 @@ class HTTPServer(TCPServer, httputil.HTTPServerConnectionDelegate):
         self.xheaders = xheaders
         self.protocol = protocol
         self.conn_params = HTTP1ConnectionParameters(
-            use_gzip=gzip,
+            decompress=decompress_request,
             chunk_size=chunk_size,
             max_header_size=max_header_size,
             header_timeout=idle_connection_timeout or 3600,

libs/tornado/httputil.py

@@ -319,7 +319,7 @@ class HTTPServerRequest(object):
        are typically kept open in HTTP/1.1, multiple requests can be handled
        sequentially on a single connection.
 
-    .. versionchanged:: 3.3
+    .. versionchanged:: 4.0
        Moved from ``tornado.httpserver.HTTPRequest``.
     """
     def __init__(self, method=None, uri=None, version="HTTP/1.0", headers=None,
@@ -352,7 +352,7 @@ class HTTPServerRequest(object):
     def supports_http_1_1(self):
         """Returns True if this request supports HTTP/1.1 semantics.
 
-        .. deprecated:: 3.3
+        .. deprecated:: 4.0
            Applications are less likely to need this information with the
            introduction of `.HTTPConnection`.  If you still need it, access
            the ``version`` attribute directly.
@@ -375,7 +375,7 @@ class HTTPServerRequest(object):
     def write(self, chunk, callback=None):
         """Writes the given chunk to the response stream.
 
-        .. deprecated:: 3.3
+        .. deprecated:: 4.0
            Use ``request.connection`` and the `.HTTPConnection` methods
            to write the response.
         """
@@ -385,7 +385,7 @@ class HTTPServerRequest(object):
     def finish(self):
         """Finishes this HTTP request on the open connection.
 
-        .. deprecated:: 3.3
+        .. deprecated:: 4.0
            Use ``request.connection`` and the `.HTTPConnection` methods
            to write the response.
         """
@@ -445,19 +445,19 @@ class HTTPServerRequest(object):
             self.__class__.__name__, args, dict(self.headers))
 
 
-class HTTPInputException(Exception):
+class HTTPInputError(Exception):
     """Exception class for malformed HTTP requests or responses
     from remote sources.
 
-    .. versionadded:: 3.3
+    .. versionadded:: 4.0
     """
     pass
 
 
-class HTTPOutputException(Exception):
+class HTTPOutputError(Exception):
     """Exception class for errors in HTTP output.
 
-    .. versionadded:: 3.3
+    .. versionadded:: 4.0
     """
     pass
 
@@ -465,7 +465,7 @@ class HTTPOutputException(Exception):
 class HTTPServerConnectionDelegate(object):
     """Implement this interface to handle requests from `.HTTPServer`.
 
-    .. versionadded:: 3.3
+    .. versionadded:: 4.0
     """
     def start_request(self, server_conn, request_conn):
         """This method is called by the server when a new request has started.
@@ -491,7 +491,7 @@ class HTTPServerConnectionDelegate(object):
 class HTTPMessageDelegate(object):
     """Implement this interface to handle an HTTP request or response.
 
-    .. versionadded:: 3.3
+    .. versionadded:: 4.0
     """
     def headers_received(self, start_line, headers):
         """Called when the HTTP headers have been received and parsed.
@@ -531,7 +531,7 @@ class HTTPMessageDelegate(object):
 class HTTPConnection(object):
     """Applications use this interface to write their responses.
 
-    .. versionadded:: 3.3
+    .. versionadded:: 4.0
     """
     def write_headers(self, start_line, headers, chunk=None, callback=None):
         """Write an HTTP header block.
@@ -774,9 +774,9 @@ def parse_request_start_line(line):
     try:
         method, path, version = line.split(" ")
     except ValueError:
-        raise HTTPInputException("Malformed HTTP request line")
+        raise HTTPInputError("Malformed HTTP request line")
     if not version.startswith("HTTP/"):
-        raise HTTPInputException(
+        raise HTTPInputError(
             "Malformed HTTP version in HTTP Request-Line: %r" % version)
     return RequestStartLine(method, path, version)
 
@@ -796,7 +796,7 @@ def parse_response_start_line(line):
     line = native_str(line)
     match = re.match("(HTTP/1.[01]) ([0-9]+) ([^\r]*)", line)
     if not match:
-        raise HTTPInputException("Error parsing response start line")
+        raise HTTPInputError("Error parsing response start line")
     return ResponseStartLine(match.group(1), int(match.group(2)),
                              match.group(3))
 

libs/tornado/ioloop.py

@@ -45,8 +45,7 @@ import traceback
 from tornado.concurrent import TracebackFuture, is_future
 from tornado.log import app_log, gen_log
 from tornado import stack_context
-from tornado.util import Configurable
-from tornado.util import errno_from_exception
+from tornado.util import Configurable, errno_from_exception, timedelta_to_seconds
 
 try:
     import signal
@@ -162,7 +161,7 @@ class IOLoop(Configurable):
     def clear_instance():
         """Clear the global `IOLoop` instance.
 
-        .. versionadded:: 3.3
+        .. versionadded:: 4.0
         """
         if hasattr(IOLoop, "_instance"):
             del IOLoop._instance
@@ -267,7 +266,7 @@ class IOLoop(Configurable):
 
         When an event occurs, ``handler(fd, events)`` will be run.
 
-        .. versionchanged:: 3.3
+        .. versionchanged:: 4.0
            Added the ability to pass file-like objects in addition to
            raw file descriptors.
         """
@@ -276,7 +275,7 @@ class IOLoop(Configurable):
     def update_handler(self, fd, events):
         """Changes the events we listen for ``fd``.
 
-        .. versionchanged:: 3.3
+        .. versionchanged:: 4.0
            Added the ability to pass file-like objects in addition to
            raw file descriptors.
         """
@@ -285,7 +284,7 @@ class IOLoop(Configurable):
     def remove_handler(self, fd):
         """Stop listening for events on ``fd``.
 
-        .. versionchanged:: 3.3
+        .. versionchanged:: 4.0
            Added the ability to pass file-like objects in addition to
            raw file descriptors.
         """
@@ -433,7 +432,7 @@ class IOLoop(Configurable):
         """
         return time.time()
 
-    def add_timeout(self, deadline, callback):
+    def add_timeout(self, deadline, callback, *args, **kwargs):
         """Runs the ``callback`` at the time ``deadline`` from the I/O loop.
 
         Returns an opaque handle that may be passed to
@@ -442,13 +441,59 @@ class IOLoop(Configurable):
         ``deadline`` may be a number denoting a time (on the same
         scale as `IOLoop.time`, normally `time.time`), or a
         `datetime.timedelta` object for a deadline relative to the
-        current time.
+        current time.  Since Tornado 4.0, `call_later` is a more
+        convenient alternative for the relative case since it does not
+        require a timedelta object.
 
         Note that it is not safe to call `add_timeout` from other threads.
         Instead, you must use `add_callback` to transfer control to the
         `IOLoop`'s thread, and then call `add_timeout` from there.
+
+        Subclasses of IOLoop must implement either `add_timeout` or
+        `call_at`; the default implementations of each will call
+        the other.  `call_at` is usually easier to implement, but
+        subclasses that wish to maintain compatibility with Tornado
+        versions prior to 4.0 must use `add_timeout` instead.
+
+        .. versionchanged:: 4.0
+           Now passes through ``*args`` and ``**kwargs`` to the callback.
         """
-        raise NotImplementedError()
+        if isinstance(deadline, numbers.Real):
+            return self.call_at(deadline, callback, *args, **kwargs)
+        elif isinstance(deadline, datetime.timedelta):
+            return self.call_at(self.time() + timedelta_to_seconds(deadline),
+                                callback, *args, **kwargs)
+        else:
+            raise TypeError("Unsupported deadline %r" % deadline)
+
+    def call_later(self, delay, callback, *args, **kwargs):
+        """Runs the ``callback`` after ``delay`` seconds have passed.
+
+        Returns an opaque handle that may be passed to `remove_timeout`
+        to cancel.  Note that unlike the `asyncio` method of the same
+        name, the returned object does not have a ``cancel()`` method.
+
+        See `add_timeout` for comments on thread-safety and subclassing.
+
+        .. versionadded:: 4.0
+        """
+        return self.call_at(self.time() + delay, callback, *args, **kwargs)
+
+    def call_at(self, when, callback, *args, **kwargs):
+        """Runs the ``callback`` at the absolute time designated by ``when``.
+
+        ``when`` must be a number using the same reference point as
+        `IOLoop.time`.
+
+        Returns an opaque handle that may be passed to `remove_timeout`
+        to cancel.  Note that unlike the `asyncio` method of the same
+        name, the returned object does not have a ``cancel()`` method.
+
+        See `add_timeout` for comments on thread-safety and subclassing.
+
+        .. versionadded:: 4.0
+        """
+        return self.add_timeout(when, callback, *args, **kwargs)
 
     def remove_timeout(self, timeout):
         """Cancels a pending timeout.
@@ -486,6 +531,19 @@ class IOLoop(Configurable):
         """
         raise NotImplementedError()
 
+    def spawn_callback(self, callback, *args, **kwargs):
+        """Calls the given callback on the next IOLoop iteration.
+
+        Unlike all other callback-related methods on IOLoop,
+        ``spawn_callback`` does not associate the callback with its caller's
+        ``stack_context``, so it is suitable for fire-and-forget callbacks
+        that should not interfere with the caller.
+
+        .. versionadded:: 4.0
+        """
+        with stack_context.NullContext():
+            self.add_callback(callback, *args, **kwargs)
+
     def add_future(self, future, callback):
         """Schedules a callback on the ``IOLoop`` when the given
         `.Future` is finished.
@@ -504,7 +562,13 @@ class IOLoop(Configurable):
         For use in subclasses.
         """
         try:
-            callback()
+            ret = callback()
+            if ret is not None and is_future(ret):
+                # Functions that return Futures typically swallow all
+                # exceptions and store them in the Future.  If a Future
+                # makes it out to the IOLoop, ensure its exception (if any)
+                # gets logged too.
+                self.add_future(ret, lambda f: f.result())
         except Exception:
             self.handle_callback_exception(callback)
 
@@ -534,7 +598,7 @@ class IOLoop(Configurable):
         This method is provided for use by `IOLoop` subclasses and should
         not generally be used by application code.
 
-        .. versionadded:: 3.3
+        .. versionadded:: 4.0
         """
         try:
             return fd.fileno(), fd
@@ -551,7 +615,7 @@ class IOLoop(Configurable):
         implementations of ``IOLoop.close(all_fds=True)`` and should
         not generally be used by application code.
 
-        .. versionadded:: 3.3
+        .. versionadded:: 4.0
         """
         try:
             try:
@@ -587,7 +651,7 @@ class PollIOLoop(IOLoop):
         self._thread_ident = None
         self._blocking_signal_threshold = None
         self._timeout_counter = itertools.count()
-        
+
         # Create a pipe that we send bogus data to when we want to wake
         # the I/O loop when it is idle
         self._waker = Waker()
@@ -680,19 +744,16 @@ class PollIOLoop(IOLoop):
 
         try:
             while True:
-                poll_timeout = _POLL_TIMEOUT
-
                 # Prevent IO event starvation by delaying new callbacks
                 # to the next iteration of the event loop.
                 with self._callback_lock:
                     callbacks = self._callbacks
                     self._callbacks = []
-                for callback in callbacks:
-                    self._run_callback(callback)
-                # Closures may be holding on to a lot of memory, so allow
-                # them to be freed before we go into our poll wait.
-                callbacks = callback = None
 
+                # Add any timeouts that have come due to the callback list.
+                # Do not run anything until we have determined which ones
+                # are ready, so timeouts that call add_timeout cannot
+                # schedule anything in this iteration.
                 if self._timeouts:
                     now = self.time()
                     while self._timeouts:
@@ -702,11 +763,9 @@ class PollIOLoop(IOLoop):
                             self._cancellations -= 1
                         elif self._timeouts[0].deadline <= now:
                             timeout = heapq.heappop(self._timeouts)
-                            self._run_callback(timeout.callback)
+                            callbacks.append(timeout.callback)
                             del timeout
                         else:
-                            seconds = self._timeouts[0].deadline - now
-                            poll_timeout = min(seconds, poll_timeout)
                             break
                     if (self._cancellations > 512
                             and self._cancellations > (len(self._timeouts) >> 1)):
@@ -717,10 +776,25 @@ class PollIOLoop(IOLoop):
                                           if x.callback is not None]
                         heapq.heapify(self._timeouts)
 
+                for callback in callbacks:
+                    self._run_callback(callback)
+                # Closures may be holding on to a lot of memory, so allow
+                # them to be freed before we go into our poll wait.
+                callbacks = callback = None
+
                 if self._callbacks:
                     # If any callbacks or timeouts called add_callback,
                     # we don't want to wait in poll() before we run them.
                     poll_timeout = 0.0
+                elif self._timeouts:
+                    # If there are any timeouts, schedule the first one.
+                    # Use self.time() instead of 'now' to account for time
+                    # spent running callbacks.
+                    poll_timeout = self._timeouts[0].deadline - self.time()
+                    poll_timeout = max(0, min(poll_timeout, _POLL_TIMEOUT))
+                else:
+                    # No timeouts and no callbacks, so use the default.
+                    poll_timeout = _POLL_TIMEOUT
 
                 if not self._running:
                     break
@@ -784,8 +858,11 @@ class PollIOLoop(IOLoop):
     def time(self):
         return self.time_func()
 
-    def add_timeout(self, deadline, callback):
-        timeout = _Timeout(deadline, stack_context.wrap(callback), self)
+    def call_at(self, deadline, callback, *args, **kwargs):
+        timeout = _Timeout(
+            deadline,
+            functools.partial(stack_context.wrap(callback), *args, **kwargs),
+            self)
         heapq.heappush(self._timeouts, timeout)
         return timeout
 
@@ -840,24 +917,12 @@ class _Timeout(object):
     __slots__ = ['deadline', 'callback', 'tiebreaker']
 
     def __init__(self, deadline, callback, io_loop):
-        if isinstance(deadline, numbers.Real):
-            self.deadline = deadline
-        elif isinstance(deadline, datetime.timedelta):
-            now = io_loop.time()
-            try:
-                self.deadline = now + deadline.total_seconds()
-            except AttributeError:  # py2.6
-                self.deadline = now + _Timeout.timedelta_to_seconds(deadline)
-        else:
+        if not isinstance(deadline, numbers.Real):
             raise TypeError("Unsupported deadline %r" % deadline)
+        self.deadline = deadline
         self.callback = callback
         self.tiebreaker = next(io_loop._timeout_counter)
 
-    @staticmethod
-    def timedelta_to_seconds(td):
-        """Equivalent to td.total_seconds() (introduced in python 2.7)."""
-        return (td.microseconds + (td.seconds + td.days * 24 * 3600) * 10 ** 6) / float(10 ** 6)
-
     # Comparison methods to sort by deadline, with object id as a tiebreaker
     # to guarantee a consistent ordering.  The heapq module uses __le__
     # in python2.5, and __lt__ in 2.6+ (sort() and most other comparisons
@@ -904,10 +969,11 @@ class PeriodicCallback(object):
         if not self._running:
             return
         try:
-            self.callback()
+            return self.callback()
         except Exception:
             self.io_loop.handle_callback_exception(self.callback)
-        self._schedule_next()
+        finally:
+            self._schedule_next()
 
     def _schedule_next(self):
         if self._running:

libs/tornado/iostream.py

@@ -57,11 +57,24 @@ except ImportError:
 # some they differ.
 _ERRNO_WOULDBLOCK = (errno.EWOULDBLOCK, errno.EAGAIN)
 
+if hasattr(errno, "WSAEWOULDBLOCK"):
+    _ERRNO_WOULDBLOCK += (errno.WSAEWOULDBLOCK,)
+
 # These errnos indicate that a connection has been abruptly terminated.
 # They should be caught and handled less noisily than other errors.
-_ERRNO_CONNRESET = (errno.ECONNRESET, errno.ECONNABORTED, errno.EPIPE)
+_ERRNO_CONNRESET = (errno.ECONNRESET, errno.ECONNABORTED, errno.EPIPE,
+                    errno.ETIMEDOUT)
 
+if hasattr(errno, "WSAECONNRESET"):
+    _ERRNO_CONNRESET += (errno.WSAECONNRESET, errno.WSAECONNABORTED, errno.WSAETIMEDOUT)
 
+# More non-portable errnos:
+_ERRNO_INPROGRESS = (errno.EINPROGRESS,)
+
+if hasattr(errno, "WSAEINPROGRESS"):
+    _ERRNO_INPROGRESS += (errno.WSAEINPROGRESS,)
+
+#######################################################
 class StreamClosedError(IOError):
     """Exception raised by `IOStream` methods when the stream is closed.
 
@@ -116,7 +129,7 @@ class BaseIOStream(object):
         :arg max_write_buffer_size: Amount of outgoing data to buffer;
             defaults to unlimited.
 
-        .. versionchanged:: 3.3
+        .. versionchanged:: 4.0
            Add the ``max_write_buffer_size`` parameter.  Changed default
            ``read_chunk_size`` to 64KB.
         """
@@ -203,7 +216,7 @@ class BaseIOStream(object):
         if more than ``max_bytes`` bytes have been read and the regex is
         not satisfied.
 
-        .. versionchanged:: 3.3
+        .. versionchanged:: 4.0
             Added the ``max_bytes`` argument.  The ``callback`` argument is
             now optional and a `.Future` will be returned if it is omitted.
         """
@@ -230,7 +243,7 @@ class BaseIOStream(object):
         if more than ``max_bytes`` bytes have been read and the delimiter
         is not found.
 
-        .. versionchanged:: 3.3
+        .. versionchanged:: 4.0
             Added the ``max_bytes`` argument.  The ``callback`` argument is
             now optional and a `.Future` will be returned if it is omitted.
         """
@@ -259,7 +272,7 @@ class BaseIOStream(object):
         If ``partial`` is true, the callback is run as soon as we have
         any bytes to return (but never more than ``num_bytes``)
 
-        .. versionchanged:: 3.3
+        .. versionchanged:: 4.0
             Added the ``partial`` argument.  The callback argument is now
             optional and a `.Future` will be returned if it is omitted.
         """
@@ -280,7 +293,7 @@ class BaseIOStream(object):
         If a callback is given, it will be run with the data as an argument;
         if not, this method returns a `.Future`.
 
-        .. versionchanged:: 3.3
+        .. versionchanged:: 4.0
             The callback argument is now optional and a `.Future` will
             be returned if it is omitted.
         """
@@ -308,7 +321,7 @@ class BaseIOStream(object):
         completed.  If `write` is called again before that `.Future` has
         resolved, the previous future will be orphaned and will never resolve.
 
-        .. versionchanged:: 3.3
+        .. versionchanged:: 4.0
             Now returns a `.Future` if no callback is given.
         """
         assert isinstance(data, bytes_type)
@@ -492,7 +505,7 @@ class BaseIOStream(object):
         def wrapper():
             self._pending_callbacks -= 1
             try:
-                callback(*args)
+                return callback(*args)
             except Exception:
                 app_log.error("Uncaught exception, closing connection.",
                               exc_info=True)
@@ -504,7 +517,8 @@ class BaseIOStream(object):
                 # Re-raise the exception so that IOLoop.handle_callback_exception
                 # can see it and log the error
                 raise
-            self._maybe_add_error_listener()
+            finally:
+                self._maybe_add_error_listener()
         # We schedule callbacks to be run on the next IOLoop iteration
         # rather than running them directly for several reasons:
         # * Prevents unbounded stack growth when a callback calls an
@@ -949,11 +963,19 @@ class IOStream(BaseIOStream):
 
         May only be called if the socket passed to the constructor was
         not previously connected.  The address parameter is in the
-        same format as for `socket.connect <socket.socket.connect>`,
-        i.e. a ``(host, port)`` tuple.  If ``callback`` is specified,
-        it will be called with no arguments when the connection is
-        completed; if not this method returns a `.Future` (whose result
-        after a successful connection will be the stream itself).
+        same format as for `socket.connect <socket.socket.connect>` for
+        the type of socket passed to the IOStream constructor,
+        e.g. an ``(ip, port)`` tuple.  Hostnames are accepted here,
+        but will be resolved synchronously and block the IOLoop.
+        If you have a hostname instead of an IP address, the `.TCPClient`
+        class is recommended instead of calling this method directly.
+        `.TCPClient` will do asynchronous DNS resolution and handle
+        both IPv4 and IPv6.
+
+        If ``callback`` is specified, it will be called with no
+        arguments when the connection is completed; if not this method
+        returns a `.Future` (whose result after a successful
+        connection will be the stream itself).
 
         If specified, the ``server_hostname`` parameter will be used
         in SSL connections for certificate validation (if requested in
@@ -966,8 +988,9 @@ class IOStream(BaseIOStream):
         is ready.  Calling `IOStream` read methods before the socket is
         connected works on some platforms but is non-portable.
 
-        .. versionchanged:: 3.3
+        .. versionchanged:: 4.0
             If no callback is given, returns a `.Future`.
+
         """
         self._connecting = True
         try:
@@ -980,7 +1003,7 @@ class IOStream(BaseIOStream):
             # returned immediately when attempting to connect to
             # localhost, so handle them the same way as an error
             # reported later in _handle_connect.
-            if (errno_from_exception(e) != errno.EINPROGRESS and
+            if (errno_from_exception(e) not in _ERRNO_INPROGRESS and
                     errno_from_exception(e) not in _ERRNO_WOULDBLOCK):
                 gen_log.warning("Connect error on fd %s: %s",
                                 self.socket.fileno(), e)
@@ -1021,7 +1044,7 @@ class IOStream(BaseIOStream):
         If a close callback is defined on this stream, it will be
         transferred to the new stream.
 
-        .. versionadded:: 3.3
+        .. versionadded:: 4.0
         """
         if (self._read_callback or self._read_future or
                 self._write_callback or self._write_future or

libs/tornado/log.py

@@ -179,7 +179,7 @@ class LogFormatter(logging.Formatter):
 def enable_pretty_logging(options=None, logger=None):
     """Turns on formatted logging output as configured.
 
-    This is called automaticaly by `tornado.options.parse_command_line`
+    This is called automatically by `tornado.options.parse_command_line`
     and `tornado.options.parse_config_file`.
     """
     if options is None:

libs/tornado/netutil.py

@@ -57,6 +57,9 @@ u('foo').encode('idna')
 # some they differ.
 _ERRNO_WOULDBLOCK = (errno.EWOULDBLOCK, errno.EAGAIN)
 
+if hasattr(errno, "WSAEWOULDBLOCK"):
+    _ERRNO_WOULDBLOCK += (errno.WSAEWOULDBLOCK,)
+
 
 def bind_sockets(port, address=None, family=socket.AF_UNSPEC, backlog=128, flags=None):
     """Creates listening sockets bound to the given port and address.

libs/tornado/platform/asyncio.py

@@ -13,9 +13,9 @@ from __future__ import absolute_import, division, print_function, with_statement
 import datetime
 import functools
 
-# _Timeout is used for its timedelta_to_seconds method for py26 compatibility.
-from tornado.ioloop import IOLoop, _Timeout
+from tornado.ioloop import IOLoop
 from tornado import stack_context
+from tornado.util import timedelta_to_seconds
 
 try:
     # Import the real asyncio module for py33+ first.  Older versions of the
@@ -109,21 +109,13 @@ class BaseAsyncIOLoop(IOLoop):
     def stop(self):
         self.asyncio_loop.stop()
 
-    def _run_callback(self, callback, *args, **kwargs):
-        try:
-            callback(*args, **kwargs)
-        except Exception:
-            self.handle_callback_exception(callback)
-
-    def add_timeout(self, deadline, callback):
-        if isinstance(deadline, (int, float)):
-            delay = max(deadline - self.time(), 0)
-        elif isinstance(deadline, datetime.timedelta):
-            delay = _Timeout.timedelta_to_seconds(deadline)
-        else:
-            raise TypeError("Unsupported deadline %r", deadline)
-        return self.asyncio_loop.call_later(delay, self._run_callback,
-                                            stack_context.wrap(callback))
+    def call_at(self, when, callback, *args, **kwargs):
+        # asyncio.call_at supports *args but not **kwargs, so bind them here.
+        # We do not synchronize self.time and asyncio_loop.time, so
+        # convert from absolute to relative.
+        return self.asyncio_loop.call_later(
+            max(0, when - self.time()), self._run_callback,
+            functools.partial(stack_context.wrap(callback), *args, **kwargs))
 
     def remove_timeout(self, timeout):
         timeout.cancel()
@@ -131,13 +123,9 @@ class BaseAsyncIOLoop(IOLoop):
     def add_callback(self, callback, *args, **kwargs):
         if self.closing:
             raise RuntimeError("IOLoop is closing")
-        if kwargs:
-            self.asyncio_loop.call_soon_threadsafe(functools.partial(
-                self._run_callback, stack_context.wrap(callback),
-                *args, **kwargs))
-        else:
-            self.asyncio_loop.call_soon_threadsafe(
-                self._run_callback, stack_context.wrap(callback), *args)
+        self.asyncio_loop.call_soon_threadsafe(
+            self._run_callback,
+            functools.partial(stack_context.wrap(callback), *args, **kwargs))
 
     add_callback_from_signal = add_callback
 

libs/tornado/platform/twisted.py

@@ -68,6 +68,7 @@ from __future__ import absolute_import, division, print_function, with_statement
 
 import datetime
 import functools
+import numbers
 import socket
 
 import twisted.internet.abstract
@@ -90,11 +91,7 @@ from tornado.log import app_log
 from tornado.netutil import Resolver
 from tornado.stack_context import NullContext, wrap
 from tornado.ioloop import IOLoop
-
-try:
-    long  # py2
-except NameError:
-    long = int  # py3
+from tornado.util import timedelta_to_seconds
 
 
 @implementer(IDelayedCall)
@@ -475,28 +472,28 @@ class TwistedIOLoop(tornado.ioloop.IOLoop):
     def stop(self):
         self.reactor.crash()
 
-    def _run_callback(self, callback, *args, **kwargs):
-        try:
-            callback(*args, **kwargs)
-        except Exception:
-            self.handle_callback_exception(callback)
-
-    def add_timeout(self, deadline, callback):
-        if isinstance(deadline, (int, long, float)):
+    def add_timeout(self, deadline, callback, *args, **kwargs):
+        # This method could be simplified (since tornado 4.0) by
+        # overriding call_at instead of add_timeout, but we leave it
+        # for now as a test of backwards-compatibility.
+        if isinstance(deadline, numbers.Real):
             delay = max(deadline - self.time(), 0)
         elif isinstance(deadline, datetime.timedelta):
-            delay = tornado.ioloop._Timeout.timedelta_to_seconds(deadline)
+            delay = timedelta_to_seconds(deadline)
         else:
             raise TypeError("Unsupported deadline %r")
-        return self.reactor.callLater(delay, self._run_callback, wrap(callback))
+        return self.reactor.callLater(
+            delay, self._run_callback,
+            functools.partial(wrap(callback), *args, **kwargs))
 
     def remove_timeout(self, timeout):
         if timeout.active():
             timeout.cancel()
 
     def add_callback(self, callback, *args, **kwargs):
-        self.reactor.callFromThread(self._run_callback,
-                                    wrap(callback), *args, **kwargs)
+        self.reactor.callFromThread(
+            self._run_callback,
+            functools.partial(wrap(callback), *args, **kwargs))
 
     def add_callback_from_signal(self, callback, *args, **kwargs):
         self.add_callback(callback, *args, **kwargs)

libs/tornado/simple_httpclient.py

@@ -277,7 +277,7 @@ class _HTTPConnection(httputil.HTTPMessageDelegate):
             stream.close()
             return
         self.stream = stream
-        self.stream.set_close_callback(self._on_close)
+        self.stream.set_close_callback(self.on_connection_close)
         self._remove_timeout()
         if self.final_callback is None:
             return
@@ -338,7 +338,7 @@ class _HTTPConnection(httputil.HTTPMessageDelegate):
         if (self.request.method == "POST" and
                 "Content-Type" not in self.request.headers):
             self.request.headers["Content-Type"] = "application/x-www-form-urlencoded"
-        if self.request.use_gzip:
+        if self.request.decompress_response:
             self.request.headers["Accept-Encoding"] = "gzip"
         req_path = ((self.parsed.path or '/') +
                     (('?' + self.parsed.query) if self.parsed.query else ''))
@@ -348,7 +348,7 @@ class _HTTPConnection(httputil.HTTPMessageDelegate):
             HTTP1ConnectionParameters(
                 no_keep_alive=True,
                 max_header_size=self.max_header_size,
-                use_gzip=self.request.use_gzip),
+                decompress=self.request.decompress_response),
             self._sockaddr)
         start_line = httputil.RequestStartLine(self.request.method,
                                                req_path, 'HTTP/1.1')
@@ -418,12 +418,15 @@ class _HTTPConnection(httputil.HTTPMessageDelegate):
             # pass it along, unless it's just the stream being closed.
             return isinstance(value, StreamClosedError)
 
-    def _on_close(self):
+    def on_connection_close(self):
         if self.final_callback is not None:
             message = "Connection closed"
             if self.stream.error:
                 raise self.stream.error
-            raise HTTPError(599, message)
+            try:
+                raise HTTPError(599, message)
+            except HTTPError:
+                self._handle_exception(*sys.exc_info())
 
     def headers_received(self, first_line, headers):
         if self.request.expect_100_continue and first_line.code == 100:
@@ -433,20 +436,6 @@ class _HTTPConnection(httputil.HTTPMessageDelegate):
         self.code = first_line.code
         self.reason = first_line.reason
 
-        if "Content-Length" in self.headers:
-            if "," in self.headers["Content-Length"]:
-                # Proxies sometimes cause Content-Length headers to get
-                # duplicated.  If all the values are identical then we can
-                # use them but if they differ it's an error.
-                pieces = re.split(r',\s*', self.headers["Content-Length"])
-                if any(i != pieces[0] for i in pieces):
-                    raise ValueError("Multiple unequal Content-Lengths: %r" %
-                                     self.headers["Content-Length"])
-                self.headers["Content-Length"] = pieces[0]
-            content_length = int(self.headers["Content-Length"])
-        else:
-            content_length = None
-
         if self.request.header_callback is not None:
             # Reassemble the start line.
             self.request.header_callback('%s %s %s\r\n' % first_line)
@@ -454,14 +443,6 @@ class _HTTPConnection(httputil.HTTPMessageDelegate):
                 self.request.header_callback("%s: %s\r\n" % (k, v))
             self.request.header_callback('\r\n')
 
-        if 100 <= self.code < 200 or self.code == 204:
-            # These response codes never have bodies
-            # http://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.3
-            if ("Transfer-Encoding" in self.headers or
-                    content_length not in (None, 0)):
-                raise ValueError("Response with code %d should not have body" %
-                                 self.code)
-
     def finish(self):
         data = b''.join(self.chunks)
         self._remove_timeout()

libs/tornado/testing.py

@@ -70,8 +70,8 @@ def get_unused_port():
     only that a series of get_unused_port calls in a single process return
     distinct ports.
 
-    **Deprecated**.  Use bind_unused_port instead, which is guaranteed
-    to find an unused port.
+    .. deprecated::
+       Use bind_unused_port instead, which is guaranteed to find an unused port.
     """
     global _next_port
     port = _next_port
@@ -459,7 +459,7 @@ def gen_test(func=None, timeout=None):
        The ``timeout`` argument and ``ASYNC_TEST_TIMEOUT`` environment
        variable.
 
-    .. versionchanged:: 3.3
+    .. versionchanged:: 4.0
        The wrapper now passes along ``*args, **kwargs`` so it can be used
        on functions with arguments.
     """

libs/tornado/util.py

@@ -311,6 +311,11 @@ class ArgReplacer(object):
         return old_value, args, kwargs
 
 
+def timedelta_to_seconds(td):
+    """Equivalent to td.total_seconds() (introduced in python 2.7)."""
+    return (td.microseconds + (td.seconds + td.days * 24 * 3600) * 10 ** 6) / float(10 ** 6)
+
+
 def _websocket_mask_python(mask, data):
     """Websocket masking function.
 

libs/tornado/web.py

@@ -35,8 +35,7 @@ Here is a simple "Hello, world" example app::
         application.listen(8888)
         tornado.ioloop.IOLoop.instance().start()
 
-See the :doc:`Tornado overview <overview>` for more details and a good getting
-started guide.
+See the :doc:`guide` for additional information.
 
 Thread-safety notes
 -------------------
@@ -48,6 +47,7 @@ not thread-safe.  In particular, methods such as
 you use multiple threads it is important to use `.IOLoop.add_callback`
 to transfer control back to the main thread before finishing the
 request.
+
 """
 
 from __future__ import absolute_import, division, print_function, with_statement
@@ -820,7 +820,7 @@ class RequestHandler(object):
         if another flush occurs before the previous flush's callback
         has been run, the previous callback will be discarded.
 
-        .. versionchanged:: 3.3
+        .. versionchanged:: 4.0
            Now returns a `.Future` if no callback is given.
         """
         chunk = b"".join(self._write_buffer)
@@ -943,26 +943,7 @@ class RequestHandler(object):
         ``kwargs["exc_info"]``.  Note that this exception may not be
         the "current" exception for purposes of methods like
         ``sys.exc_info()`` or ``traceback.format_exc``.
-
-        For historical reasons, if a method ``get_error_html`` exists,
-        it will be used instead of the default ``write_error`` implementation.
-        ``get_error_html`` returned a string instead of producing output
-        normally, and had different semantics for exception handling.
-        Users of ``get_error_html`` are encouraged to convert their code
-        to override ``write_error`` instead.
         """
-        if hasattr(self, 'get_error_html'):
-            if 'exc_info' in kwargs:
-                exc_info = kwargs.pop('exc_info')
-                kwargs['exception'] = exc_info[1]
-                try:
-                    # Put the traceback into sys.exc_info()
-                    raise_exc_info(exc_info)
-                except Exception:
-                    self.finish(self.get_error_html(status_code, **kwargs))
-            else:
-                self.finish(self.get_error_html(status_code, **kwargs))
-            return
         if self.settings.get("serve_traceback") and "exc_info" in kwargs:
             # in debug mode, try to send a traceback
             self.set_header('Content-Type', 'text/plain')
@@ -1147,14 +1128,15 @@ class RequestHandler(object):
             else:
                 # Treat unknown versions as not present instead of failing.
                 return None, None, None
-        elif len(cookie) == 32:
+        else:
             version = 1
-            token = binascii.a2b_hex(utf8(cookie))
+            try:
+                token = binascii.a2b_hex(utf8(cookie))
+            except (binascii.Error, TypeError):
+                token = utf8(cookie)
             # We don't have a usable timestamp in older versions.
             timestamp = int(time.time())
             return (version, token, timestamp)
-        else:
-            return None, None, None
 
     def check_xsrf_cookie(self):
         """Verifies that the ``_xsrf`` cookie matches the ``_xsrf`` argument.
@@ -1242,27 +1224,6 @@ class RequestHandler(object):
 
         return base + get_url(self.settings, path, **kwargs)
 
-    def async_callback(self, callback, *args, **kwargs):
-        """Obsolete - catches exceptions from the wrapped function.
-
-        This function is unnecessary since Tornado 1.1.
-        """
-        if callback is None:
-            return None
-        if args or kwargs:
-            callback = functools.partial(callback, *args, **kwargs)
-
-        def wrapper(*args, **kwargs):
-            try:
-                return callback(*args, **kwargs)
-            except Exception as e:
-                if self._headers_written:
-                    app_log.error("Exception after headers written",
-                                  exc_info=True)
-                else:
-                    self._handle_request_exception(e)
-        return wrapper
-
     def require_setting(self, name, feature="this feature"):
         """Raises an exception if the given app setting is not defined."""
         if not self.application.settings.get(name):
@@ -1405,6 +1366,11 @@ class RequestHandler(object):
             " (" + self.request.remote_ip + ")"
 
     def _handle_request_exception(self, e):
+        if isinstance(e, Finish):
+            # Not an error; just finish the request without logging.
+            if not self._finished:
+                self.finish()
+            return
         self.log_exception(*sys.exc_info())
         if self._finished:
             # Extra errors after the request has been finished should
@@ -1662,7 +1628,7 @@ class Application(httputil.HTTPServerConnectionDelegate):
                  **settings):
         if transforms is None:
             self.transforms = []
-            if settings.get("gzip"):
+            if settings.get("compress_response") or settings.get("gzip"):
                 self.transforms.append(GZipContentEncoding)
         else:
             self.transforms = transforms
@@ -1959,6 +1925,9 @@ class HTTPError(Exception):
     `RequestHandler.send_error` since it automatically ends the
     current function.
 
+    To customize the response sent with an `HTTPError`, override
+    `RequestHandler.write_error`.
+
     :arg int status_code: HTTP status code.  Must be listed in
         `httplib.responses <http.client.responses>` unless the ``reason``
         keyword argument is given.
@@ -1987,6 +1956,25 @@ class HTTPError(Exception):
             return message
 
 
+class Finish(Exception):
+    """An exception that ends the request without producing an error response.
+
+    When `Finish` is raised in a `RequestHandler`, the request will end
+    (calling `RequestHandler.finish` if it hasn't already been called),
+    but the outgoing response will not be modified and the error-handling
+    methods (including `RequestHandler.write_error`) will not be called.
+
+    This can be a more convenient way to implement custom error pages
+    than overriding ``write_error`` (especially in library code)::
+
+        if self.current_user is None:
+            self.set_status(401)
+            self.set_header('WWW-Authenticate', 'Basic realm="something"')
+            raise Finish()
+    """
+    pass
+
+
 class MissingArgumentError(HTTPError):
     """Exception raised by `RequestHandler.get_argument`.
 
@@ -2367,7 +2355,7 @@ class StaticFileHandler(RequestHandler):
 
         .. versionadded:: 3.1
 
-        .. versionchanged:: 3.3
+        .. versionchanged:: 4.0
            This method is now always called, instead of only when
            partial results are requested.
         """
@@ -2514,9 +2502,9 @@ class FallbackHandler(RequestHandler):
 class OutputTransform(object):
     """A transform modifies the result of an HTTP request (e.g., GZip encoding)
 
-    A new transform instance is created for every request. See the
-    GZipContentEncoding example below if you want to implement a
-    new Transform.
+    Applications are not expected to create their own OutputTransforms
+    or interact with them directly; the framework chooses which transforms
+    (if any) to apply.
     """
     def __init__(self, request):
         pass
@@ -2533,7 +2521,7 @@ class GZipContentEncoding(OutputTransform):
 
     See http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.11
 
-    .. versionchanged:: 3.3
+    .. versionchanged:: 4.0
         Now compresses all mime types beginning with ``text/``, instead
         of just a whitelist. (the whitelist is still used for certain
         non-text mime types).
@@ -2767,7 +2755,7 @@ class URLSpec(object):
           in the regex will be passed in to the handler's get/post/etc
           methods as arguments.
 
-        * ``handler_class``: `RequestHandler` subclass to be invoked.
+        * ``handler``: `RequestHandler` subclass to be invoked.
 
         * ``kwargs`` (optional): A dictionary of additional arguments
           to be passed to the handler's constructor.

libs/tornado/websocket.py

@@ -3,18 +3,17 @@
 `WebSockets <http://dev.w3.org/html5/websockets/>`_ allow for bidirectional
 communication between the browser and server.
 
-.. warning::
+WebSockets are supported in the current versions of all major browsers,
+although older versions that do not support WebSockets are still in use
+(refer to http://caniuse.com/websockets for details).
 
-   The WebSocket protocol was recently finalized as `RFC 6455
-   <http://tools.ietf.org/html/rfc6455>`_ and is not yet supported in
-   all browsers.  Refer to http://caniuse.com/websockets for details
-   on compatibility.  In addition, during development the protocol
-   went through several incompatible versions, and some browsers only
-   support older versions.  By default this module only supports the
-   latest version of the protocol, but optional support for an older
-   version (known as "draft 76" or "hixie-76") can be enabled by
-   overriding `WebSocketHandler.allow_draft76` (see that method's
-   documentation for caveats).
+This module implements the final version of the WebSocket protocol as
+defined in `RFC 6455 <http://tools.ietf.org/html/rfc6455>`_.  Certain
+browser versions (notably Safari 5.x) implemented an earlier draft of
+the protocol (known as "draft 76") and are not compatible with this module.
+
+.. versionchanged:: 4.0
+   Removed support for the draft 76 protocol version.
 """
 
 from __future__ import absolute_import, division, print_function, with_statement
@@ -22,11 +21,9 @@ from __future__ import absolute_import, division, print_function, with_statement
 
 import base64
 import collections
-import functools
 import hashlib
 import os
 import struct
-import time
 import tornado.escape
 import tornado.web
 
@@ -38,7 +35,7 @@ from tornado.iostream import StreamClosedError
 from tornado.log import gen_log, app_log
 from tornado import simple_httpclient
 from tornado.tcpclient import TCPClient
-from tornado.util import bytes_type, unicode_type, _websocket_mask
+from tornado.util import bytes_type, _websocket_mask
 
 try:
     from urllib.parse import urlparse # py2
@@ -108,6 +105,21 @@ class WebSocketHandler(tornado.web.RequestHandler):
       };
 
     This script pops up an alert box that says "You said: Hello, world".
+
+    Web browsers allow any site to open a websocket connection to any other,
+    instead of using the same-origin policy that governs other network
+    access from javascript.  This can be surprising and is a potential
+    security hole, so since Tornado 4.0 `WebSocketHandler` requires
+    applications that wish to receive cross-origin websockets to opt in
+    by overriding the `~WebSocketHandler.check_origin` method (see that
+    method's docs for details).  Failure to do so is the most likely
+    cause of 403 errors when making a websocket connection.
+
+    When using a secure websocket connection (``wss://``) with a self-signed
+    certificate, the connection from a browser may fail because it wants
+    to show the "accept this certificate" dialog but has nowhere to show it.
+    You must first visit a regular HTML page using the same certificate
+    to accept it before the websocket connection will succeed.
     """
     def __init__(self, application, request, **kwargs):
         tornado.web.RequestHandler.__init__(self, application, request,
@@ -115,22 +127,17 @@ class WebSocketHandler(tornado.web.RequestHandler):
         self.ws_connection = None
         self.close_code = None
         self.close_reason = None
+        self.stream = None
 
     @tornado.web.asynchronous
     def get(self, *args, **kwargs):
         self.open_args = args
         self.open_kwargs = kwargs
 
-        self.stream = self.request.connection.detach()
-        self.stream.set_close_callback(self.on_connection_close)
-
         # Upgrade header should be present and should be equal to WebSocket
         if self.request.headers.get("Upgrade", "").lower() != 'websocket':
-            self.stream.write(tornado.escape.utf8(
-                "HTTP/1.1 400 Bad Request\r\n\r\n"
-                "Can \"Upgrade\" only to \"WebSocket\"."
-            ))
-            self.stream.close()
+            self.set_status(400)
+            self.finish("Can \"Upgrade\" only to \"WebSocket\".")
             return
 
         # Connection header should be upgrade. Some proxy servers/load balancers
@@ -138,11 +145,8 @@ class WebSocketHandler(tornado.web.RequestHandler):
         headers = self.request.headers
         connection = map(lambda s: s.strip().lower(), headers.get("Connection", "").split(","))
         if 'upgrade' not in connection:
-            self.stream.write(tornado.escape.utf8(
-                "HTTP/1.1 400 Bad Request\r\n\r\n"
-                "\"Connection\" must be \"Upgrade\"."
-            ))
-            self.stream.close()
+            self.set_status(400)
+            self.finish("\"Connection\" must be \"Upgrade\".")
             return
 
         # Handle WebSocket Origin naming convention differences
@@ -159,19 +163,16 @@ class WebSocketHandler(tornado.web.RequestHandler):
         # according to check_origin. When the origin is None, we assume it
         # did not come from a browser and that it can be passed on.
         if origin is not None and not self.check_origin(origin):
-            self.stream.write(tornado.escape.utf8(
-                "HTTP/1.1 403 Cross Origin Websockets Disabled\r\n\r\n"
-            ))
-            self.stream.close()
+            self.set_status(403)
+            self.finish("Cross origin websockets not allowed")
             return
 
+        self.stream = self.request.connection.detach()
+        self.stream.set_close_callback(self.on_connection_close)
+
         if self.request.headers.get("Sec-WebSocket-Version") in ("7", "8", "13"):
             self.ws_connection = WebSocketProtocol13(self)
             self.ws_connection.accept_connection()
-        elif (self.allow_draft76() and
-              "Sec-WebSocket-Version" not in self.request.headers):
-            self.ws_connection = WebSocketProtocol76(self)
-            self.ws_connection.accept_connection()
         else:
             self.stream.write(tornado.escape.utf8(
                 "HTTP/1.1 426 Upgrade Required\r\n"
@@ -245,7 +246,7 @@ class WebSocketHandler(tornado.web.RequestHandler):
         phrase was supplied, these values will be available as the attributes
         ``self.close_code`` and ``self.close_reason``.
 
-        .. versionchanged:: 3.3
+        .. versionchanged:: 4.0
 
            Added ``close_code`` and ``close_reason`` attributes.
         """
@@ -263,10 +264,7 @@ class WebSocketHandler(tornado.web.RequestHandler):
         closing.  These values are made available to the client, but are
         not otherwise interpreted by the websocket protocol.
 
-        The ``code`` and ``reason`` arguments are ignored in the "draft76"
-        protocol version.
-
-        .. versionchanged:: 3.3
+        .. versionchanged:: 4.0
 
            Added the ``code`` and ``reason`` arguments.
         """
@@ -292,7 +290,20 @@ class WebSocketHandler(tornado.web.RequestHandler):
         browsers, since WebSockets are allowed to bypass the usual same-origin
         policies and don't use CORS headers.
 
-        .. versionadded:: 3.3
+        To accept all cross-origin traffic (which was the default prior to
+        Tornado 4.0), simply override this method to always return true::
+
+            def check_origin(self, origin):
+                return True
+
+        To allow connections from any subdomain of your site, you might
+        do something like::
+
+            def check_origin(self, origin):
+                parsed_origin = urllib.parse.urlparse(origin)
+                return parsed_origin.netloc.endswith(".mydomain.com")
+
+        .. versionadded:: 4.0
         """
         parsed_origin = urlparse(origin)
         origin = parsed_origin.netloc
@@ -303,21 +314,6 @@ class WebSocketHandler(tornado.web.RequestHandler):
         # Check to see that origin matches host directly, including ports
         return origin == host
 
-    def allow_draft76(self):
-        """Override to enable support for the older "draft76" protocol.
-
-        The draft76 version of the websocket protocol is disabled by
-        default due to security concerns, but it can be enabled by
-        overriding this method to return True.
-
-        Connections using the draft76 protocol do not support the
-        ``binary=True`` flag to `write_message`.
-
-        Support for the draft76 protocol is deprecated and will be
-        removed in a future version of Tornado.
-        """
-        return False
-
     def set_nodelay(self, value):
         """Set the no-delay flag for this stream.
 
@@ -334,29 +330,6 @@ class WebSocketHandler(tornado.web.RequestHandler):
         """
         self.stream.set_nodelay(value)
 
-    def get_websocket_scheme(self):
-        """Return the url scheme used for this request, either "ws" or "wss".
-
-        This is normally decided by HTTPServer, but applications
-        may wish to override this if they are using an SSL proxy
-        that does not provide the X-Scheme header as understood
-        by HTTPServer.
-
-        Note that this is only used by the draft76 protocol.
-        """
-        return "wss" if self.request.protocol == "https" else "ws"
-
-    def async_callback(self, callback, *args, **kwargs):
-        """Obsolete - catches exceptions from the wrapped function.
-
-        This function is normally unncecessary thanks to
-        `tornado.stack_context`.
-        """
-        return self.ws_connection.async_callback(callback, *args, **kwargs)
-
-    def _not_supported(self, *args, **kwargs):
-        raise Exception("Method not supported for Web Sockets")
-
     def on_connection_close(self):
         if self.ws_connection:
             self.ws_connection.on_connection_close()
@@ -364,9 +337,17 @@ class WebSocketHandler(tornado.web.RequestHandler):
             self.on_close()
 
 
+def _wrap_method(method):
+    def _disallow_for_websocket(self, *args, **kwargs):
+        if self.stream is None:
+            method(self, *args, **kwargs)
+        else:
+            raise RuntimeError("Method not supported for Web Sockets")
+    return _disallow_for_websocket
 for method in ["write", "redirect", "set_header", "send_error", "set_cookie",
                "set_status", "flush", "finish"]:
-    setattr(WebSocketHandler, method, WebSocketHandler._not_supported)
+    setattr(WebSocketHandler, method,
+            _wrap_method(getattr(WebSocketHandler, method)))
 
 
 class WebSocketProtocol(object):
@@ -379,23 +360,17 @@ class WebSocketProtocol(object):
         self.client_terminated = False
         self.server_terminated = False
 
-    def async_callback(self, callback, *args, **kwargs):
-        """Wrap callbacks with this if they are used on asynchronous requests.
+    def _run_callback(self, callback, *args, **kwargs):
+        """Runs the given callback with exception handling.
 
-        Catches exceptions properly and closes this WebSocket if an exception
-        is uncaught.
+        On error, aborts the websocket connection and returns False.
         """
-        if args or kwargs:
-            callback = functools.partial(callback, *args, **kwargs)
-
-        def wrapper(*args, **kwargs):
-            try:
-                return callback(*args, **kwargs)
-            except Exception:
-                app_log.error("Uncaught exception in %s",
-                              self.request.path, exc_info=True)
-                self._abort()
-        return wrapper
+        try:
+            callback(*args, **kwargs)
+        except Exception:
+            app_log.error("Uncaught exception in %s",
+                          self.request.path, exc_info=True)
+            self._abort()
 
     def on_connection_close(self):
         self._abort()
@@ -408,174 +383,6 @@ class WebSocketProtocol(object):
         self.close()  # let the subclass cleanup
 
 
-class WebSocketProtocol76(WebSocketProtocol):
-    """Implementation of the WebSockets protocol, version hixie-76.
-
-    This class provides basic functionality to process WebSockets requests as
-    specified in
-    http://tools.ietf.org/html/draft-hixie-thewebsocketprotocol-76
-    """
-    def __init__(self, handler):
-        WebSocketProtocol.__init__(self, handler)
-        self.challenge = None
-        self._waiting = None
-
-    def accept_connection(self):
-        try:
-            self._handle_websocket_headers()
-        except ValueError:
-            gen_log.debug("Malformed WebSocket request received")
-            self._abort()
-            return
-
-        scheme = self.handler.get_websocket_scheme()
-
-        # draft76 only allows a single subprotocol
-        subprotocol_header = ''
-        subprotocol = self.request.headers.get("Sec-WebSocket-Protocol", None)
-        if subprotocol:
-            selected = self.handler.select_subprotocol([subprotocol])
-            if selected:
-                assert selected == subprotocol
-                subprotocol_header = "Sec-WebSocket-Protocol: %s\r\n" % selected
-
-        # Write the initial headers before attempting to read the challenge.
-        # This is necessary when using proxies (such as HAProxy), which
-        # need to see the Upgrade headers before passing through the
-        # non-HTTP traffic that follows.
-        self.stream.write(tornado.escape.utf8(
-            "HTTP/1.1 101 WebSocket Protocol Handshake\r\n"
-            "Upgrade: WebSocket\r\n"
-            "Connection: Upgrade\r\n"
-            "Server: TornadoServer/%(version)s\r\n"
-            "Sec-WebSocket-Origin: %(origin)s\r\n"
-            "Sec-WebSocket-Location: %(scheme)s://%(host)s%(uri)s\r\n"
-            "%(subprotocol)s"
-            "\r\n" % (dict(
-                version=tornado.version,
-                origin=self.request.headers["Origin"],
-                scheme=scheme,
-                host=self.request.host,
-                uri=self.request.uri,
-                subprotocol=subprotocol_header))))
-        self.stream.read_bytes(8, self._handle_challenge)
-
-    def challenge_response(self, challenge):
-        """Generates the challenge response that's needed in the handshake
-
-        The challenge parameter should be the raw bytes as sent from the
-        client.
-        """
-        key_1 = self.request.headers.get("Sec-Websocket-Key1")
-        key_2 = self.request.headers.get("Sec-Websocket-Key2")
-        try:
-            part_1 = self._calculate_part(key_1)
-            part_2 = self._calculate_part(key_2)
-        except ValueError:
-            raise ValueError("Invalid Keys/Challenge")
-        return self._generate_challenge_response(part_1, part_2, challenge)
-
-    def _handle_challenge(self, challenge):
-        try:
-            challenge_response = self.challenge_response(challenge)
-        except ValueError:
-            gen_log.debug("Malformed key data in WebSocket request")
-            self._abort()
-            return
-        self._write_response(challenge_response)
-
-    def _write_response(self, challenge):
-        self.stream.write(challenge)
-        self.async_callback(self.handler.open)(*self.handler.open_args, **self.handler.open_kwargs)
-        self._receive_message()
-
-    def _handle_websocket_headers(self):
-        """Verifies all invariant- and required headers
-
-        If a header is missing or have an incorrect value ValueError will be
-        raised
-        """
-        fields = ("Origin", "Host", "Sec-Websocket-Key1",
-                  "Sec-Websocket-Key2")
-        if not all(map(lambda f: self.request.headers.get(f), fields)):
-            raise ValueError("Missing/Invalid WebSocket headers")
-
-    def _calculate_part(self, key):
-        """Processes the key headers and calculates their key value.
-
-        Raises ValueError when feed invalid key."""
-        # pyflakes complains about variable reuse if both of these lines use 'c'
-        number = int(''.join(c for c in key if c.isdigit()))
-        spaces = len([c2 for c2 in key if c2.isspace()])
-        try:
-            key_number = number // spaces
-        except (ValueError, ZeroDivisionError):
-            raise ValueError
-        return struct.pack(">I", key_number)
-
-    def _generate_challenge_response(self, part_1, part_2, part_3):
-        m = hashlib.md5()
-        m.update(part_1)
-        m.update(part_2)
-        m.update(part_3)
-        return m.digest()
-
-    def _receive_message(self):
-        self.stream.read_bytes(1, self._on_frame_type)
-
-    def _on_frame_type(self, byte):
-        frame_type = ord(byte)
-        if frame_type == 0x00:
-            self.stream.read_until(b"\xff", self._on_end_delimiter)
-        elif frame_type == 0xff:
-            self.stream.read_bytes(1, self._on_length_indicator)
-        else:
-            self._abort()
-
-    def _on_end_delimiter(self, frame):
-        if not self.client_terminated:
-            self.async_callback(self.handler.on_message)(
-                frame[:-1].decode("utf-8", "replace"))
-        if not self.client_terminated:
-            self._receive_message()
-
-    def _on_length_indicator(self, byte):
-        if ord(byte) != 0x00:
-            self._abort()
-            return
-        self.client_terminated = True
-        self.close()
-
-    def write_message(self, message, binary=False):
-        """Sends the given message to the client of this Web Socket."""
-        if binary:
-            raise ValueError(
-                "Binary messages not supported by this version of websockets")
-        if isinstance(message, unicode_type):
-            message = message.encode("utf-8")
-        assert isinstance(message, bytes_type)
-        self.stream.write(b"\x00" + message + b"\xff")
-
-    def write_ping(self, data):
-        """Send ping frame."""
-        raise ValueError("Ping messages not supported by this version of websockets")
-
-    def close(self, code=None, reason=None):
-        """Closes the WebSocket connection."""
-        if not self.server_terminated:
-            if not self.stream.closed():
-                self.stream.write("\xff\x00")
-            self.server_terminated = True
-        if self.client_terminated:
-            if self._waiting is not None:
-                self.stream.io_loop.remove_timeout(self._waiting)
-            self._waiting = None
-            self.stream.close()
-        elif self._waiting is None:
-            self._waiting = self.stream.io_loop.add_timeout(
-                time.time() + 5, self._abort)
-
-
 class WebSocketProtocol13(WebSocketProtocol):
     """Implementation of the WebSocket protocol from RFC 6455.
 
@@ -645,7 +452,8 @@ class WebSocketProtocol13(WebSocketProtocol):
             "%s"
             "\r\n" % (self._challenge_response(), subprotocol_header)))
 
-        self.async_callback(self.handler.open)(*self.handler.open_args, **self.handler.open_kwargs)
+        self._run_callback(self.handler.open, *self.handler.open_args,
+                           **self.handler.open_kwargs)
         self._receive_frame()
 
     def _write_frame(self, fin, opcode, data):
@@ -803,10 +611,10 @@ class WebSocketProtocol13(WebSocketProtocol):
             except UnicodeDecodeError:
                 self._abort()
                 return
-            self.async_callback(self.handler.on_message)(decoded)
+            self._run_callback(self.handler.on_message, decoded)
         elif opcode == 0x2:
             # Binary data
-            self.async_callback(self.handler.on_message)(data)
+            self._run_callback(self.handler.on_message, data)
         elif opcode == 0x8:
             # Close
             self.client_terminated = True
@@ -820,7 +628,7 @@ class WebSocketProtocol13(WebSocketProtocol):
             self._write_frame(True, 0xA, data)
         elif opcode == 0xA:
             # Pong
-            self.async_callback(self.handler.on_pong)(data)
+            self._run_callback(self.handler.on_pong, data)
         else:
             self._abort()
 
@@ -885,7 +693,7 @@ class WebSocketClientConnection(simple_httpclient._HTTPConnection):
 
         .. versionadded:: 3.2
 
-        .. versionchanged:: 3.3
+        .. versionchanged:: 4.0
 
            Added the ``code`` and ``reason`` arguments.
         """
@@ -893,10 +701,12 @@ class WebSocketClientConnection(simple_httpclient._HTTPConnection):
             self.protocol.close(code, reason)
             self.protocol = None
 
-    def _on_close(self):
+    def on_connection_close(self):
+        if not self.connect_future.done():
+            self.connect_future.set_exception(StreamClosedError())
         self.on_message(None)
-        self.resolver.close()
-        super(WebSocketClientConnection, self)._on_close()
+        self.tcp_client.close()
+        super(WebSocketClientConnection, self).on_connection_close()
 
     def _on_http_response(self, response):
         if not self.connect_future.done():
@@ -925,7 +735,12 @@ class WebSocketClientConnection(simple_httpclient._HTTPConnection):
             self._timeout = None
 
         self.stream = self.connection.detach()
-        self.stream.set_close_callback(self._on_close)
+        self.stream.set_close_callback(self.on_connection_close)
+        # Once we've taken over the connection, clear the final callback
+        # we set on the http request.  This deactivates the error handling
+        # in simple_httpclient that would otherwise interfere with our
+        # ability to see exceptions.
+        self.final_callback = None
 
         self.connect_future.set_result(self)
 

libs/tornado/wsgi.py

@@ -77,7 +77,7 @@ else:
 class WSGIApplication(web.Application):
     """A WSGI equivalent of `tornado.web.Application`.
 
-    .. deprecated: 3.3::
+    .. deprecated:: 4.0
 
        Use a regular `.Application` and wrap it in `WSGIAdapter` instead.
     """
@@ -126,7 +126,7 @@ class _WSGIConnection(httputil.HTTPConnection):
         if self._expected_content_remaining is not None:
             self._expected_content_remaining -= len(chunk)
             if self._expected_content_remaining < 0:
-                self._error = httputil.HTTPOutputException(
+                self._error = httputil.HTTPOutputError(
                     "Tried to write more data than Content-Length")
                 raise self._error
         self._write_buffer.append(chunk)
@@ -137,7 +137,7 @@ class _WSGIConnection(httputil.HTTPConnection):
     def finish(self):
         if (self._expected_content_remaining is not None and
                 self._expected_content_remaining != 0):
-            self._error = httputil.HTTPOutputException(
+            self._error = httputil.HTTPOutputError(
                 "Tried to write %d bytes less than Content-Length" %
                 self._expected_content_remaining)
             raise self._error
@@ -183,7 +183,7 @@ class WSGIAdapter(object):
     that it is not possible to use `.AsyncHTTPClient`, or the
     `tornado.auth` or `tornado.websocket` modules.
 
-    .. versionadded:: 3.3
+    .. versionadded:: 4.0
     """
     def __init__(self, application):
         if isinstance(application, WSGIApplication):