Merge branch 'refs/heads/develop'

contribute.md

@@ -0,0 +1 @@
+So you feel like

couchpotato/core/downloaders/sabnzbd/main.py

@@ -2,6 +2,7 @@ from couchpotato.core.downloaders.base import Downloader
 from couchpotato.core.helpers.encoding import tryUrlencode
 from couchpotato.core.helpers.variable import cleanHost, mergeDicts
 from couchpotato.core.logger import CPLog
+from urllib2 import URLError
 import json
 import traceback
 
@@ -36,31 +37,31 @@ class Sabnzbd(Downloader):
         else:
             params['name'] = data.get('url')
 
-        url = cleanHost(self.conf('host')) + "api?" + tryUrlencode(params)
+        url = cleanHost(self.conf('host')) + 'api?' + tryUrlencode(params)
 
         try:
             if params.get('mode') is 'addfile':
-                sab = self.urlopen(url, timeout = 60, params = {"nzbfile": (nzb_filename, filedata)}, multipart = True, show_error = False)
+                sab = self.urlopen(url, timeout = 60, params = {'nzbfile': (nzb_filename, filedata)}, multipart = True, show_error = False)
             else:
                 sab = self.urlopen(url, timeout = 60, show_error = False)
+        except URLError:
+            log.error('Failed sending release, probably wrong HOST: %s', traceback.format_exc(0))
+            return False
         except:
-            log.error('Failed sending release: %s', traceback.format_exc())
+            log.error('Failed sending release, use API key, NOT the NZB key: %s', traceback.format_exc(0))
             return False
 
         result = sab.strip()
         if not result:
-            log.error("SABnzbd didn't return anything.")
+            log.error('SABnzbd didn\'t return anything.')
             return False
 
-        log.debug("Result text from SAB: " + result[:40])
-        if result == "ok":
-            log.info("NZB sent to SAB successfully.")
+        log.debug('Result text from SAB: %s', result[:40])
+        if result[:2] == 'ok':
+            log.info('NZB sent to SAB successfully.')
             return True
-        elif result == "Missing authentication":
-            log.error("Incorrect username/password.")
-            return False
         else:
-            log.error("Unknown error: " + result[:40])
+            log.error(result[:40])
             return False
 
     def getAllDownloadStatus(self):
@@ -75,7 +76,7 @@ class Sabnzbd(Downloader):
                 'mode': 'queue',
             })
         except:
-            log.error('Failed getting queue: %s', traceback.format_exc())
+            log.error('Failed getting queue: %s', traceback.format_exc(0))
             return False
 
         # Go through history items
@@ -85,7 +86,7 @@ class Sabnzbd(Downloader):
                 'limit': 15,
             })
         except:
-            log.error('Failed getting history json: %s', traceback.format_exc())
+            log.error('Failed getting history json: %s', traceback.format_exc(0))
             return False
 
         statuses = []
@@ -134,21 +135,26 @@ class Sabnzbd(Downloader):
                 'value': item['id']
             }, use_json = False)
         except:
-            log.error('Failed deleting: %s', traceback.format_exc())
+            log.error('Failed deleting: %s', traceback.format_exc(0))
             return False
 
         return True
 
     def call(self, params, use_json = True):
 
-        url = cleanHost(self.conf('host')) + "api?" + tryUrlencode(mergeDicts(params, {
+        url = cleanHost(self.conf('host')) + 'api?' + tryUrlencode(mergeDicts(params, {
            'apikey': self.conf('api_key'),
            'output': 'json'
         }))
 
         data = self.urlopen(url, timeout = 60, show_error = False)
         if use_json:
-            return json.loads(data)[params['mode']]
+            d = json.loads(data)
+            if d.get('error'):
+                log.error('Error getting data from SABNZBd: %s', d.get('error'))
+                return {}
+
+            return d[params['mode']]
         else:
             return data
 

couchpotato/core/event.py

@@ -19,7 +19,7 @@ def addEvent(name, handler, priority = 100):
     if events.get(name):
         e = events[name]
     else:
-        e = events[name] = Event(name = name, threads = 20, exc_info = True, traceback = True, lock = threading.RLock())
+        e = events[name] = Event(name = name, threads = 10, exc_info = True, traceback = True, lock = threading.RLock())
 
     def createHandle(*args, **kwargs):
 

couchpotato/core/helpers/variable.py

@@ -135,3 +135,5 @@ def getTitle(library_dict):
 def randomString(size = 8, chars = string.ascii_uppercase + string.digits):
     return ''.join(random.choice(chars) for x in range(size))
 
+def splitString(str, split_on = ','):
+    return [x.strip() for x in str.split(split_on)]

couchpotato/core/notifications/base.py

@@ -14,7 +14,7 @@ class Notification(Plugin):
     test_message = 'ZOMG Lazors Pewpewpew!'
 
     listen_to = [
-        'movie.downloaded', 'movie.snatched',
+        'renamer.after', 'movie.snatched',
         'updater.available', 'updater.updated',
     ]
     dont_listen_to = []
@@ -30,10 +30,10 @@ class Notification(Plugin):
                 addEvent(listener, self.createNotifyHandler(listener))
 
     def createNotifyHandler(self, listener):
-        def notify(message, data):
+        def notify(message = None, group = {}, data = None):
             if not self.conf('on_snatch', default = True) and listener == 'movie.snatched':
                 return
-            return self.notify(message = message, data = data, listener = listener)
+            return self.notify(message = message, data = data if data else group, listener = listener)
 
         return notify
 

couchpotato/core/notifications/core/main.py

@@ -3,7 +3,7 @@ from couchpotato.api import addApiView, addNonBlockApiView
 from couchpotato.core.event import addEvent
 from couchpotato.core.helpers.encoding import toUnicode
 from couchpotato.core.helpers.request import jsonified, getParam
-from couchpotato.core.helpers.variable import tryInt
+from couchpotato.core.helpers.variable import tryInt, splitString
 from couchpotato.core.logger import CPLog
 from couchpotato.core.notifications.base import Notification
 from couchpotato.core.settings.model import Notification as Notif
@@ -22,7 +22,7 @@ class CoreNotifier(Notification):
     listeners = []
 
     listen_to = [
-        'movie.downloaded', 'movie.snatched',
+        'renamer.after', 'movie.snatched',
         'updater.available', 'updater.updated',
     ]
 
@@ -67,7 +67,7 @@ class CoreNotifier(Notification):
 
         ids = None
         if getParam('ids'):
-            ids = [x.strip() for x in getParam('ids').split(',')]
+            ids = splitString(getParam('ids'))
 
         db = get_session()
 
@@ -92,7 +92,7 @@ class CoreNotifier(Notification):
         q = db.query(Notif)
 
         if limit_offset:
-            splt = [x.strip() for x in limit_offset.split(',')]
+            splt = splitString(limit_offset)
             limit = splt[0]
             offset = 0 if len(splt) is 1 else splt[1]
             q = q.limit(limit).offset(offset)

couchpotato/core/notifications/nmj/main.py

@@ -69,7 +69,7 @@ class NMJ(Notification):
             'mount': mount,
         })
 
-    def addToLibrary(self, group = {}):
+    def addToLibrary(self, message = None, group = {}):
         if self.isDisabled(): return
 
         host = self.conf('host')
@@ -114,8 +114,8 @@ class NMJ(Notification):
 
     def failed(self):
         return jsonified({'success': False})
-        
+
     def test(self):
         return jsonified({'success': self.addToLibrary()})
-    
-    
+
+

couchpotato/core/notifications/notifymyandroid/main.py

@@ -1,3 +1,4 @@
+from couchpotato.core.helpers.variable import splitString
 from couchpotato.core.logger import CPLog
 from couchpotato.core.notifications.base import Notification
 import pynma
@@ -11,7 +12,7 @@ class NotifyMyAndroid(Notification):
         if self.isDisabled(): return
 
         nma = pynma.PyNMA()
-        keys = [x.strip() for x in self.conf('api_key').split(',')]
+        keys = splitString(self.conf('api_key'))
         nma.addkey(keys)
         nma.developerkey(self.conf('dev_key'))
 

couchpotato/core/notifications/notifymywp/main.py

@@ -1,3 +1,4 @@
+from couchpotato.core.helpers.variable import splitString
 from couchpotato.core.logger import CPLog
 from couchpotato.core.notifications.base import Notification
 from pynmwp import PyNMWP
@@ -10,7 +11,7 @@ class NotifyMyWP(Notification):
     def notify(self, message = '', data = {}, listener = None):
         if self.isDisabled(): return
 
-        keys = [x.strip() for x in self.conf('api_key').split(',')]
+        keys = splitString(self.conf('api_key'))
         p = PyNMWP(keys, self.conf('dev_key'))
 
         response = p.push(application = self.default_title, event = message, description = message, priority = self.conf('priority'), batch_mode = len(keys) > 1)

couchpotato/core/notifications/plex/main.py

@@ -17,7 +17,7 @@ class Plex(Notification):
         super(Plex, self).__init__()
         addEvent('renamer.after', self.addToLibrary)
 
-    def addToLibrary(self, group = {}):
+    def addToLibrary(self, message = None, group = {}):
         if self.isDisabled(): return
 
         log.info('Sending notification to Plex')

couchpotato/core/notifications/prowl/main.py

@@ -1,39 +1,35 @@
-from couchpotato.core.helpers.encoding import toUnicode, tryUrlencode
+from couchpotato.core.helpers.encoding import toUnicode
 from couchpotato.core.logger import CPLog
 from couchpotato.core.notifications.base import Notification
-from httplib import HTTPSConnection
+import traceback
 
 log = CPLog(__name__)
 
 
 class Prowl(Notification):
 
+    urls = {
+        'api': 'https://api.prowlapp.com/publicapi/add'
+    }
+
     def notify(self, message = '', data = {}, listener = None):
         if self.isDisabled(): return
 
-        http_handler = HTTPSConnection('api.prowlapp.com')
-
         data = {
             'apikey': self.conf('api_key'),
             'application': self.default_title,
             'description': toUnicode(message),
             'priority': self.conf('priority'),
         }
+        headers = {
+           'Content-type': 'application/x-www-form-urlencoded'
+        }
 
-        http_handler.request('POST',
-            '/publicapi/add',
-            headers = {'Content-type': 'application/x-www-form-urlencoded'},
-            body = tryUrlencode(data)
-        )
-        response = http_handler.getresponse()
-        request_status = response.status
-
-        if request_status == 200:
+        try:
+            self.urlopen(self.urls['api'], headers = headers, params = data, multipart = True, show_error = False)
             log.info('Prowl notifications sent.')
             return True
-        elif request_status == 401:
-            log.error('Prowl auth failed: %s', response.reason)
-            return False
-        else:
-            log.error('Prowl notification failed.')
-            return False
+        except:
+            log.error('Prowl failed: %s', traceback.format_exc())
+
+        return False

couchpotato/core/notifications/synoindex/main.py

@@ -16,7 +16,7 @@ class Synoindex(Notification):
         super(Synoindex, self).__init__()
         addEvent('renamer.after', self.addToLibrary)
 
-    def addToLibrary(self, group = {}):
+    def addToLibrary(self, message = None, group = {}):
         if self.isDisabled(): return
 
         command = [self.index_path, '-A', group.get('destination_dir')]

couchpotato/core/notifications/twitter/main.py

@@ -38,22 +38,33 @@ class Twitter(Notification):
 
         direct_message = self.conf('direct_message')
         direct_message_users = self.conf('screen_name')
-        
+
         mention = self.conf('mention')
+        mention_tag = None
         if mention:
             if direct_message:
                 direct_message_users = '%s %s' % (direct_message_users, mention)
-                direct_message_users = direct_message_users.replace('@',' ')
-                direct_message_users = direct_message_users.replace(',',' ')
+                direct_message_users = direct_message_users.replace('@', ' ')
+                direct_message_users = direct_message_users.replace(',', ' ')
             else:
-                message = '%s @%s' % (message, mention.lstrip('@'))
+                mention_tag = '@%s' % mention.lstrip('@')
+                message = '%s %s' % (message, mention_tag)
 
         try:
             if direct_message:
                 for user in direct_message_users.split():
                     api.PostDirectMessage(user, '[%s] %s' % (self.default_title, message))
             else:
-                api.PostUpdate('[%s] %s' % (self.default_title, message))
+                update_message = '[%s] %s' % (self.default_title, message)
+                if len(update_message) > 140:
+                    if mention_tag:
+                        api.PostUpdate(update_message[:135 - len(mention_tag)] + ('%s 1/2 ' % mention_tag))
+                        api.PostUpdate(update_message[135 - len(mention_tag):] + ('%s 2/2 ' % mention_tag))
+                    else:
+                        api.PostUpdate(update_message[:135] + ' 1/2')
+                        api.PostUpdate(update_message[135:] + ' 2/2')
+                else:
+                    api.PostUpdate(update_message)
         except Exception, e:
             log.error('Error sending tweet: %s', e)
             return False

couchpotato/core/notifications/xbmc/main.py

@@ -1,6 +1,7 @@
-from couchpotato.core.helpers.encoding import tryUrlencode
+from couchpotato.core.helpers.variable import splitString
 from couchpotato.core.logger import CPLog
 from couchpotato.core.notifications.base import Notification
+from flask.helpers import json
 import base64
 
 log = CPLog(__name__)
@@ -8,36 +9,51 @@ log = CPLog(__name__)
 
 class XBMC(Notification):
 
-    listen_to = ['movie.downloaded']
+    listen_to = ['renamer.after']
 
     def notify(self, message = '', data = {}, listener = None):
         if self.isDisabled(): return
 
-        hosts = [x.strip() for x in self.conf('host').split(",")]
+        hosts = splitString(self.conf('host'))
         successful = 0
         for host in hosts:
-            if self.send({'command': 'ExecBuiltIn', 'parameter': 'Notification(CouchPotato, %s)' % message}, host):
-                successful += 1
-            if self.send({'command': 'ExecBuiltIn', 'parameter': 'XBMC.updatelibrary(video)'}, host):
-                successful += 1
+            response = self.request(host, [
+                ('GUI.ShowNotification', {"title":"CouchPotato", "message":message}),
+                ('VideoLibrary.Scan', {}),
+            ])
 
-        return successful == len(hosts)*2
+            for result in response:
+                if result['result'] == "OK":
+                    successful += 1
 
-    def send(self, command, host):
+        return successful == len(hosts) * 2
 
-        url = 'http://%s/xbmcCmds/xbmcHttp/?%s' % (host, tryUrlencode(command))
+    def request(self, host, requests):
+        server = 'http://%s/jsonrpc' % host
+
+        data = []
+        for req in requests:
+            method, kwargs = req
+            data.append({
+                'method': method,
+                'params': kwargs,
+                'jsonrpc': '2.0',
+                'id': method,
+            })
+        data = json.dumps(data)
+
+        headers = {
+            'Content-Type': 'application/json',
+        }
 
-        headers = {}
         if self.conf('password'):
-            headers = {
-               'Authorization': "Basic %s" % base64.encodestring('%s:%s' % (self.conf('username'), self.conf('password')))[:-1]
-            }
+            base64string = base64.encodestring('%s:%s' % (self.conf('username'), self.conf('password'))).replace('\n', '')
+            headers['Authorization'] = 'Basic %s' % base64string
 
-        try:
-            self.urlopen(url, headers = headers, show_error = False)
-        except:
-            log.error("Couldn't sent command to XBMC")
-            return False
+        log.debug('Sending request to %s: %s', (host, data))
+        rdata = self.urlopen(server, headers = headers, params = data, multipart = True)
+        response = json.loads(rdata)
+        log.debug('Returned from request %s: %s', (host, response))
+
+        return response
 
-        log.info('XBMC notification to %s successful.', host)
-        return True

couchpotato/core/plugins/automation/main.py

@@ -18,9 +18,17 @@ class Automation(Plugin):
     def addMovies(self):
 
         movies = fireEvent('automation.get_movies', merge = True)
+        movie_ids = []
+
         for imdb_id in movies:
             prop_name = 'automation.added.%s' % imdb_id
             added = Env.prop(prop_name, default = False)
             if not added:
-                fireEvent('movie.add', params = {'identifier': imdb_id}, force_readd = False)
+                added_movie = fireEvent('movie.add', params = {'identifier': imdb_id}, force_readd = False, search_after = False, update_library = True, single = True)
+                if added_movie:
+                    movie_ids.append(added_movie['id'])
                 Env.prop(prop_name, True)
+
+        for movie_id in movie_ids:
+            movie_dict = fireEvent('movie.get', movie_id, single = True)
+            fireEvent('searcher.single', movie_dict)

couchpotato/core/plugins/base.py

@@ -124,7 +124,7 @@ class Plugin(object):
         try:
 
             if multipart:
-                log.info('Opening multipart url: %s, params: %s', (url, [x for x in params.iterkeys()]))
+                log.info('Opening multipart url: %s, params: %s', (url, [x for x in params.iterkeys()] if isinstance(params, dict) else 'with data'))
                 request = urllib2.Request(url, params, headers)
 
                 cookies = cookielib.CookieJar()
@@ -239,7 +239,8 @@ class Plugin(object):
                     self.setCache(cache_key, data, timeout = cache_timeout)
                 return data
             except:
-                pass
+                if not kwargs.get('show_error'):
+                    raise
 
     def setCache(self, cache_key, value, timeout = 300):
         log.debug('Setting cache %s', cache_key)

couchpotato/core/plugins/file/main.py

@@ -2,9 +2,11 @@ from couchpotato import get_session
 from couchpotato.api import addApiView
 from couchpotato.core.event import addEvent
 from couchpotato.core.helpers.encoding import toUnicode
+from couchpotato.core.helpers.request import jsonified
 from couchpotato.core.helpers.variable import md5, getExt
 from couchpotato.core.logger import CPLog
 from couchpotato.core.plugins.base import Plugin
+from couchpotato.core.plugins.scanner.main import Scanner
 from couchpotato.core.settings.model import FileType, File
 from couchpotato.environment import Env
 import os.path
@@ -29,7 +31,34 @@ class FileManager(Plugin):
             'return': {'type': 'file'}
         })
 
+        addApiView('file.types', self.getTypesView, docs = {
+            'desc': 'Return a list of all the file types and their ids.',
+            'return': {'type': 'object', 'example': """{
+    'types': [
+        {
+            "identifier": "poster_original",
+            "type": "image",
+            "id": 1,
+            "name": "Poster_original"
+        },
+        {
+            "identifier": "poster",
+            "type": "image",
+            "id": 2,
+            "name": "Poster"
+        },
+        etc
+    ]
+}"""}
+        })
+
         addEvent('app.load', self.cleanup)
+        addEvent('app.load', self.init)
+
+    def init(self):
+
+        for type_tuple in Scanner.file_types.values():
+            self.getType(type_tuple)
 
     def cleanup(self):
 
@@ -122,3 +151,9 @@ class FileManager(Plugin):
             types.append(type_object.to_dict())
 
         return types
+
+    def getTypesView(self):
+
+        return jsonified({
+            'types': self.getTypes()
+        })

couchpotato/core/plugins/manage/main.py

@@ -2,22 +2,32 @@ from couchpotato.api import addApiView
 from couchpotato.core.event import fireEvent, addEvent, fireEventAsync
 from couchpotato.core.helpers.encoding import ss
 from couchpotato.core.helpers.request import jsonified, getParam
+from couchpotato.core.helpers.variable import getTitle, splitString
 from couchpotato.core.logger import CPLog
 from couchpotato.core.plugins.base import Plugin
 from couchpotato.environment import Env
 import os
 import time
+import traceback
 
 
 log = CPLog(__name__)
 
 class Manage(Plugin):
 
+    in_progress = False
+
     def __init__(self):
 
         fireEvent('scheduler.interval', identifier = 'manage.update_library', handle = self.updateLibrary, hours = 2)
 
         addEvent('manage.update', self.updateLibrary)
+
+        # Add files after renaming
+        def after_rename(message = None, group = {}):
+            return self.scanFilesToLibrary(folder = group['destination_dir'], files = group['renamed_files'])
+        addEvent('renamer.after', after_rename, priority = 110)
+
         addApiView('manage.update', self.updateLibraryView, docs = {
             'desc': 'Update the library by scanning for new movies',
             'params': {
@@ -25,11 +35,23 @@ class Manage(Plugin):
             }
         })
 
+        addApiView('manage.progress', self.getProgress, docs = {
+            'desc': 'Get the progress of current manage update',
+            'return': {'type': 'object', 'example': """{
+    'progress': False || object, total & to_go,
+}"""},
+        })
+
         if not Env.get('dev'):
             def updateLibrary():
                 self.updateLibrary(full = False)
             addEvent('app.load', updateLibrary)
 
+    def getProgress(self):
+        return jsonified({
+            'progress': self.in_progress
+        })
+
     def updateLibraryView(self):
 
         full = getParam('full', default = 1)
@@ -43,49 +65,127 @@ class Manage(Plugin):
     def updateLibrary(self, full = True):
         last_update = float(Env.prop('manage.last_update', default = 0))
 
-        if self.isDisabled() or (last_update > time.time() - 20):
+        if self.in_progress:
+            log.info('Already updating library: %s', self.in_progress)
+            return
+        elif self.isDisabled() or (last_update > time.time() - 20):
             return
 
-        directories = self.directories()
-        added_identifiers = []
+        self.in_progress = {}
+        fireEvent('notify.frontend', type = 'manage.updating', data = True)
 
-        for directory in directories:
+        try:
 
-            if not os.path.isdir(directory):
-                if len(directory) > 0:
-                    log.error('Directory doesn\'t exist: %s', directory)
-                continue
+            directories = self.directories()
+            added_identifiers = []
 
-            log.info('Updating manage library: %s', directory)
-            identifiers = fireEvent('scanner.folder', folder = directory, newer_than = last_update if not full else 0, single = True)
-            if identifiers:
-                added_identifiers.extend(identifiers)
+            # Add some progress
+            self.in_progress = {}
+            for directory in directories:
+                self.in_progress[os.path.normpath(directory)] = {
+                    'total': None,
+                    'to_go': None,
+                }
 
-            # Break if CP wants to shut down
-            if self.shuttingDown():
+            for directory in directories:
+                folder = os.path.normpath(directory)
+
+                if not os.path.isdir(folder):
+                    if len(directory) > 0:
+                        log.error('Directory doesn\'t exist: %s', folder)
+                    continue
+
+                log.info('Updating manage library: %s', folder)
+                fireEvent('notify.frontend', type = 'manage.update', data = True, message = 'Scanning for movies in "%s"' % folder)
+
+                onFound = self.createAddToLibrary(folder, added_identifiers)
+                fireEvent('scanner.scan', folder = folder, simple = True, newer_than = last_update if not full else 0, on_found = onFound, single = True)
+
+                # Break if CP wants to shut down
+                if self.shuttingDown():
+                    break
+
+            # If cleanup option is enabled, remove offline files from database
+            if self.conf('cleanup') and full and not self.shuttingDown():
+
+                # Get movies with done status
+                total_movies, done_movies = fireEvent('movie.list', status = 'done', single = True)
+
+                for done_movie in done_movies:
+                    if done_movie['library']['identifier'] not in added_identifiers:
+                        fireEvent('movie.delete', movie_id = done_movie['id'], delete_from = 'all')
+                    else:
+                        for release in done_movie.get('releases', []):
+                            for release_file in release.get('files', []):
+                                # Remove release not available anymore
+                                if not os.path.isfile(ss(release_file['path'])):
+                                    fireEvent('release.clean', release['id'])
+                                    break
+
+            Env.prop('manage.last_update', time.time())
+        except:
+            log.error('Failed updating library: %s', (traceback.format_exc()))
+
+        while True and not self.shuttingDown():
+
+            delete_me = {}
+
+            for folder in self.in_progress:
+                if self.in_progress[folder]['to_go'] <= 0:
+                    delete_me[folder] = True
+
+            for delete in delete_me:
+                del self.in_progress[delete]
+
+            if len(self.in_progress) == 0:
                 break
 
-        # If cleanup option is enabled, remove offline files from database
-        if self.conf('cleanup') and full and not self.shuttingDown():
+            time.sleep(1)
 
-            # Get movies with done status
-            total_movies, done_movies = fireEvent('movie.list', status = 'done', single = True)
+        fireEvent('notify.frontend', type = 'manage.updating', data = False)
+        self.in_progress = False
 
-            for done_movie in done_movies:
-                if done_movie['library']['identifier'] not in added_identifiers:
-                    fireEvent('movie.delete', movie_id = done_movie['id'], delete_from = 'all')
-                else:
-                    for release in done_movie.get('releases', []):
-                        for release_file in release.get('files', []):
-                            # Remove release not available anymore
-                            if not os.path.isfile(ss(release_file['path'])):
-                                fireEvent('release.clean', release['id'])
-                                break
+    def createAddToLibrary(self, folder, added_identifiers = []):
+        def addToLibrary(group, total_found, to_go):
+            if self.in_progress[folder]['total'] is None:
+                self.in_progress[folder] = {
+                    'total': total_found,
+                    'to_go': total_found,
+                }
 
-        Env.prop('manage.last_update', time.time())
+            if group['library']:
+                identifier = group['library'].get('identifier')
+                added_identifiers.append(identifier)
+
+                # Add it to release and update the info
+                fireEvent('release.add', group = group)
+                fireEventAsync('library.update', identifier = identifier, on_complete = self.createAfterUpdate(folder, identifier))
+
+        return addToLibrary
+
+    def createAfterUpdate(self, folder, identifier):
+
+        # Notify frontend
+        def afterUpdate():
+            self.in_progress[folder]['to_go'] = self.in_progress[folder]['to_go'] - 1
+            total = self.in_progress[folder]['total']
+            movie_dict = fireEvent('movie.get', identifier, single = True)
+            fireEvent('notify.frontend', type = 'movie.added', data = movie_dict, message = None if total > 5 else 'Added "%s" to manage.' % getTitle(movie_dict['library']))
+
+        return afterUpdate
 
     def directories(self):
         try:
-            return [x.strip() for x in self.conf('library', default = '').split('::')]
+            return splitString(self.conf('library', default = ''), '::')
         except:
             return []
+
+    def scanFilesToLibrary(self, folder = None, files = None):
+
+        folder = os.path.normpath(folder)
+
+        groups = fireEvent('scanner.scan', folder = folder, files = files, single = True)
+
+        for group in groups.itervalues():
+            if group['library']:
+                fireEvent('release.add', group = group)

couchpotato/core/plugins/movie/main.py

@@ -3,7 +3,7 @@ from couchpotato.api import addApiView
 from couchpotato.core.event import fireEvent, fireEventAsync, addEvent
 from couchpotato.core.helpers.encoding import toUnicode, simplifyString
 from couchpotato.core.helpers.request import getParams, jsonified, getParam
-from couchpotato.core.helpers.variable import getImdb
+from couchpotato.core.helpers.variable import getImdb, splitString
 from couchpotato.core.logger import CPLog
 from couchpotato.core.plugins.base import Plugin
 from couchpotato.core.settings.model import Library, LibraryTitle, Movie
@@ -107,13 +107,18 @@ class MoviePlugin(Plugin):
     def get(self, movie_id):
 
         db = get_session()
-        m = db.query(Movie).filter_by(id = movie_id).first()
+
+        imdb_id = getImdb(str(movie_id))
+
+        if(imdb_id):
+            m = db.query(Movie).filter(Movie.library.has(identifier = imdb_id)).first()
+        else:
+            m = db.query(Movie).filter_by(id = movie_id).first()
 
         results = None
         if m:
             results = m.to_dict(self.default_dict)
 
-        #db.close()
         return results
 
     def list(self, status = ['active'], limit_offset = None, starts_with = None, search = None):
@@ -161,7 +166,7 @@ class MoviePlugin(Plugin):
             .options(joinedload_all('files'))
 
         if limit_offset:
-            splt = [x.strip() for x in limit_offset.split(',')]
+            splt = splitString(limit_offset)
             limit = splt[0]
             offset = 0 if len(splt) is 1 else splt[1]
             q2 = q2.limit(limit).offset(offset)
@@ -239,7 +244,7 @@ class MoviePlugin(Plugin):
 
         db = get_session()
 
-        for id in getParam('id').split(','):
+        for id in splitString(getParam('id')):
             movie = db.query(Movie).filter_by(id = id).first()
 
             if movie:
@@ -278,7 +283,7 @@ class MoviePlugin(Plugin):
             'movies': movies,
         })
 
-    def add(self, params = {}, force_readd = True, search_after = True):
+    def add(self, params = {}, force_readd = True, search_after = True, update_library = False):
 
         if not params.get('identifier'):
             msg = 'Can\'t add movie without imdb identifier.'
@@ -298,7 +303,7 @@ class MoviePlugin(Plugin):
                 pass
 
 
-        library = fireEvent('library.add', single = True, attrs = params, update_after = False)
+        library = fireEvent('library.add', single = True, attrs = params, update_after = update_library)
 
         # Status
         status_active = fireEvent('status.add', 'active', single = True)
@@ -381,7 +386,7 @@ class MoviePlugin(Plugin):
 
         available_status = fireEvent('status.get', 'available', single = True)
 
-        ids = [x.strip() for x in params.get('id').split(',')]
+        ids = splitString(params.get('id'))
         for movie_id in ids:
 
             m = db.query(Movie).filter_by(id = movie_id).first()
@@ -417,7 +422,7 @@ class MoviePlugin(Plugin):
 
         params = getParams()
 
-        ids = [x.strip() for x in params.get('id').split(',')]
+        ids = splitString(params.get('id'))
         for movie_id in ids:
             self.delete(movie_id, delete_from = params.get('delete_from', 'all'))
 

couchpotato/core/plugins/movie/static/list.js

@@ -33,29 +33,34 @@ var MovieList = new Class({
 		);
 		self.getMovies();
 
-		if(options.add_new)
-			App.addEvent('movie.added', self.movieAdded.bind(self))
-
+		App.addEvent('movie.added', self.movieAdded.bind(self))
 		App.addEvent('movie.deleted', self.movieDeleted.bind(self))
 	},
 
 	movieDeleted: function(notification){
 		var self = this;
 
-		if(!self.movies_added[notification.data.id])
-			self.movies_added[notification.data.id].destroy();
+		if(self.movies_added[notification.data.id]){
+			self.movies.each(function(movie){
+				if(movie.get('id') == notification.data.id){
+					movie.destroy();
+					delete self.movies_added[notification.data.id]
+				}
+			})
+		}
 
 		self.checkIfEmpty();
 	},
 
 	movieAdded: function(notification){
 		var self = this;
-		window.scroll(0,0);
 
-		if(!self.movies_added[notification.data.id])
+		if(self.options.add_new && !self.movies_added[notification.data.id] && notification.data.status.identifier == self.options.status){
+			window.scroll(0,0);
 			self.createMovie(notification.data, 'top');
 
-		self.checkIfEmpty();
+			self.checkIfEmpty();
+		}
 	},
 
 	create: function(){
@@ -100,6 +105,7 @@ var MovieList = new Class({
 			self.createMovie(movie);
 		});
 
+		self.total_movies = total;
 		self.setCounter(total);
 
 	},
@@ -495,7 +501,7 @@ var MovieList = new Class({
 	checkIfEmpty: function(){
 		var self = this;
 
-		var is_empty = self.movies.length == 0;
+		var is_empty = self.movies.length == 0 && self.total_movies == 0;
 
 		if(is_empty && self.options.on_empty_element){
 			self.el.grab(self.options.on_empty_element);

couchpotato/core/plugins/movie/static/movie.css

@@ -24,7 +24,7 @@
 		.movies .movie.list_view:hover, .movies .movie.mass_edit_view:hover {
 			background: rgba(255,255,255,0.03);
 		}
-		
+
 		.movies .movie_container {
 			overflow: hidden;
 		}
@@ -313,7 +313,7 @@
 					padding-bottom: 4px;
 					height: auto;
 				}
-				
+
 				.movies .movie .trailer_container {
 					width: 100%;
 					background: #000;
@@ -324,7 +324,7 @@
 					.movies .movie .trailer_container.hide {
 						height: 0 !important;
 					}
-				
+
 					.movies .movie .hide_trailer {
 						position: absolute;
 						top: 0;
@@ -340,12 +340,12 @@
 					.movies .movie .hide_trailer.hide {
 						top: -30px;
 					}
-						
+
 				.movies .movie .try_container {
 					padding: 5px 10px;
 					text-align: center;
 				}
-						
+
 					.movies .movie .try_container a {
 						margin: 0 5px;
 						padding: 2px 5px;
@@ -354,15 +354,15 @@
 					.movies .movie .releases .next_release {
 						border-left: 6px solid #2aa300;
 					}
-					
+
 						.movies .movie .releases .next_release > :first-child {
 							margin-left: -6px;
 						}
-						
+
 					.movies .movie .releases .last_release {
 						border-left: 6px solid #ffa200;
 					}
-					
+
 						.movies .movie .releases .last_release > :first-child {
 							margin-left: -6px;
 						}
@@ -394,8 +394,8 @@
 		border-radius: 0;
 	}
 
-.movies .alph_nav ul.numbers, 
-.movies .alph_nav .counter, 
+.movies .alph_nav ul.numbers,
+.movies .alph_nav .counter,
 .movies .alph_nav ul.actions {
 	list-style: none;
 	padding: 0 0 1px;
@@ -518,7 +518,7 @@
 				padding: 3px 7px;
 			}
 
-		.movies .alph_nav .mass_edit_form .refresh, 
+		.movies .alph_nav .mass_edit_form .refresh,
 		.movies .alph_nav .mass_edit_form .delete {
 			float: left;
 			padding: 8px 0 0 8px;
@@ -536,3 +536,65 @@
 		.movies .alph_nav .more_menu > a {
 			background-position: center -158px;
 		}
+
+.movies .empty_wanted {
+	background-image: url('../images/emptylist.png');
+	height: 750px;
+	width: 800px;
+	padding-top: 260px;
+	margin-top: -50px;
+}
+
+.movies .empty_manage {
+	text-align: center;
+	font-size: 25px;
+	line-height: 150%;
+}
+
+	.movies .empty_manage .after_manage {
+		margin-top: 30px;
+		font-size: 16px;
+	}
+
+	.movies .progress {
+		border-radius: 2px;
+		padding: 10px;
+		margin: 5px 0;
+		text-align: left;
+	}
+
+		.movies .progress > div {
+			padding: 5px 10px;
+			font-size: 12px;
+			line-height: 12px;
+			text-align: left;
+			display: inline-block;
+			width: 49%;
+			background: rgba(255, 255, 255, 0.05);
+			margin: 2px 0.5%;
+			border-radius: 3px;
+		}
+
+			.movies .progress > div .folder {
+				display: inline-block;
+				padding: 5px 20px 5px 0;
+			    white-space: nowrap;
+				text-overflow: ellipsis;
+				overflow: hidden;
+				width: 85%;
+				direction: rtl;
+				vertical-align: middle;
+			}
+
+			.movies .progress > div .percentage {
+				font-weight: bold;
+				display: inline-block;
+				text-transform: uppercase;
+				text-shadow: none;
+				font-weight: normal;
+				font-size: 20px;
+				border-left: 1px solid rgba(255, 255, 255, .2);
+				width: 15%;
+				text-align: right;
+				vertical-align: middle;
+			}

couchpotato/core/plugins/renamer/main.py

@@ -50,7 +50,7 @@ class Renamer(Plugin):
             return
 
         if self.renaming_started is True:
-            log.info('Renamer is disabled to avoid infinite looping of the same error.')
+            log.info('Renamer is already running, if you see this often, check the logs above for errors.')
             return
 
         # Check to see if the "to" folder is inside the "from" folder.
@@ -129,6 +129,8 @@ class Renamer(Plugin):
                      'resolution_width': group['meta_data'].get('resolution_width'),
                      'resolution_height': group['meta_data'].get('resolution_height'),
                      'imdb_id': library['identifier'],
+                     'cd': '',
+                     'cd_nr': '',
                 }
 
                 for file_type in group['files']:
@@ -146,7 +148,7 @@ class Renamer(Plugin):
                         continue
 
                     # Move other files
-                    multiple = len(group['files']['movie']) > 1 and not group['is_dvd']
+                    multiple = len(group['files'][file_type]) > 1 and not group['is_dvd']
                     cd = 1 if multiple else 0
 
                     for current_file in sorted(list(group['files'][file_type])):
@@ -159,23 +161,19 @@ class Renamer(Plugin):
                         replacements['ext'] = getExt(current_file)
 
                         # cd #
-                        replacements['cd'] = ' cd%d' % cd if cd else ''
-                        replacements['cd_nr'] = cd
+                        replacements['cd'] = ' cd%d' % cd if multiple else ''
+                        replacements['cd_nr'] = cd if multiple else ''
 
                         # Naming
                         final_folder_name = self.doReplace(folder_name, replacements)
                         final_file_name = self.doReplace(file_name, replacements)
                         replacements['filename'] = final_file_name[:-(len(getExt(final_file_name)) + 1)]
 
-                        # Group filename without cd extension
-                        replacements['cd'] = ''
-                        replacements['cd_nr'] = ''
-
                         # Meta naming
                         if file_type is 'trailer':
-                            final_file_name = self.doReplace(trailer_name, replacements)
+                            final_file_name = self.doReplace(trailer_name, replacements, remove_multiple = True)
                         elif file_type is 'nfo':
-                            final_file_name = self.doReplace(nfo_name, replacements)
+                            final_file_name = self.doReplace(nfo_name, replacements, remove_multiple = True)
 
                         # Seperator replace
                         if separator:
@@ -206,10 +204,16 @@ class Renamer(Plugin):
                         # Check for extra subtitle files
                         if file_type is 'subtitle':
 
-                            # rename subtitles with or without language
-                            rename_files[current_file] = os.path.join(destination, final_folder_name, final_file_name)
+                            remove_multiple = False
+                            if len(group['files']['movie']) == 1:
+                                remove_multiple = True
+
                             sub_langs = group['subtitle_language'].get(current_file, [])
 
+                            # rename subtitles with or without language
+                            sub_name = self.doReplace(file_name, replacements, remove_multiple = remove_multiple)
+                            rename_files[current_file] = os.path.join(destination, final_folder_name, sub_name)
+
                             rename_extras = self.getRenameExtras(
                                 extra_type = 'subtitle_extra',
                                 replacements = replacements,
@@ -217,20 +221,19 @@ class Renamer(Plugin):
                                 file_name = file_name,
                                 destination = destination,
                                 group = group,
-                                current_file = current_file
+                                current_file = current_file,
+                                remove_multiple = remove_multiple,
                             )
 
-                            # Don't add language if multiple languages in 1 file
-                            if len(sub_langs) > 1:
-                                rename_files[current_file] = os.path.join(destination, final_folder_name, final_file_name)
-                            elif len(sub_langs) == 1:
+                            # Don't add language if multiple languages in 1 subtitle file
+                            if len(sub_langs) == 1:
                                 sub_name = final_file_name.replace(replacements['ext'], '%s.%s' % (sub_langs[0], replacements['ext']))
                                 rename_files[current_file] = os.path.join(destination, final_folder_name, sub_name)
 
                             rename_files = mergeDicts(rename_files, rename_extras)
 
                         # Filename without cd etc
-                        if file_type is 'movie':
+                        elif file_type is 'movie':
                             rename_extras = self.getRenameExtras(
                                 extra_type = 'movie_extra',
                                 replacements = replacements,
@@ -242,7 +245,7 @@ class Renamer(Plugin):
                             )
                             rename_files = mergeDicts(rename_files, rename_extras)
 
-                            group['filename'] = self.doReplace(file_name, replacements)[:-(len(getExt(final_file_name)) + 1)]
+                            group['filename'] = self.doReplace(file_name, replacements, remove_multiple = True)[:-(len(getExt(final_file_name)) + 1)]
                             group['destination_dir'] = os.path.join(destination, final_folder_name)
 
                         if multiple:
@@ -377,12 +380,9 @@ class Renamer(Plugin):
                 except:
                     log.error('Failed removing %s: %s', (group['parentdir'], traceback.format_exc()))
 
-            # Search for trailers etc
-            fireEventAsync('renamer.after', group)
-
-            # Notify on download
+            # Notify on download, search for trailers etc
             download_message = 'Downloaded %s (%s)' % (movie_title, replacements['quality'])
-            fireEventAsync('movie.downloaded', message = download_message, data = group)
+            fireEvent('renamer.after', message = download_message, group = group, in_order = True)
 
             # Break if CP wants to shut down
             if self.shuttingDown():
@@ -390,8 +390,9 @@ class Renamer(Plugin):
 
         self.renaming_started = False
 
-    def getRenameExtras(self, extra_type = '', replacements = {}, folder_name = '', file_name = '', destination = '', group = {}, current_file = ''):
+    def getRenameExtras(self, extra_type = '', replacements = {}, folder_name = '', file_name = '', destination = '', group = {}, current_file = '', remove_multiple = False):
 
+        replacements = replacements.copy()
         rename_files = {}
 
         def test(s):
@@ -400,8 +401,8 @@ class Renamer(Plugin):
         for extra in set(filter(test, group['files'][extra_type])):
             replacements['ext'] = getExt(extra)
 
-            final_folder_name = self.doReplace(folder_name, replacements)
-            final_file_name = self.doReplace(file_name, replacements)
+            final_folder_name = self.doReplace(folder_name, replacements, remove_multiple = remove_multiple)
+            final_file_name = self.doReplace(file_name, replacements, remove_multiple = remove_multiple)
             rename_files[extra] = os.path.join(destination, final_folder_name, final_file_name)
 
         return rename_files
@@ -456,11 +457,16 @@ class Renamer(Plugin):
 
         return True
 
-    def doReplace(self, string, replacements):
+    def doReplace(self, string, replacements, remove_multiple = False):
         '''
         replace confignames with the real thing
         '''
 
+        replacements = replacements.copy()
+        if remove_multiple:
+            replacements['cd'] = ''
+            replacements['cd_nr'] = ''
+
         replaced = toUnicode(string)
         for x, r in replacements.iteritems():
             if r is not None:

couchpotato/core/plugins/scanner/main.py

@@ -34,6 +34,7 @@ class Scanner(Plugin):
         'subtitle_extra': ['idx'],
         'trailer': ['mov', 'mp4', 'flv']
     }
+
     file_types = {
         'subtitle': ('subtitle', 'subtitle'),
         'subtitle_extra': ('subtitle', 'subtitle_extra'),
@@ -42,6 +43,8 @@ class Scanner(Plugin):
         'movie': ('video', 'movie'),
         'movie_extra': ('movie', 'movie_extra'),
         'backdrop': ('image', 'backdrop'),
+        'poster': ('image', 'poster'),
+        'thumbnail': ('image', 'thumbnail'),
         'leftover': ('leftover', 'leftover'),
     }
 
@@ -80,55 +83,10 @@ class Scanner(Plugin):
         addEvent('scanner.remove_cptag', self.removeCPTag)
 
         addEvent('scanner.scan', self.scan)
-        addEvent('scanner.files', self.scanFilesToLibrary)
-        addEvent('scanner.folder', self.scanFolderToLibrary)
         addEvent('scanner.name_year', self.getReleaseNameYear)
         addEvent('scanner.partnumber', self.getPartNumber)
 
-        def after_rename(group):
-            return self.scanFilesToLibrary(folder = group['destination_dir'], files = group['renamed_files'])
-
-        addEvent('renamer.after', after_rename)
-
-    def scanFilesToLibrary(self, folder = None, files = None):
-
-        folder = os.path.normpath(folder)
-
-        groups = self.scan(folder = folder, files = files)
-
-        for group in groups.itervalues():
-            if group['library']:
-                fireEvent('release.add', group = group)
-
-    def scanFolderToLibrary(self, folder = None, newer_than = 0, simple = True):
-
-        folder = os.path.normpath(folder)
-
-        if not os.path.isdir(folder):
-            return
-
-        groups = self.scan(folder = folder, simple = simple, newer_than = newer_than)
-
-        added_identifier = []
-        while True and not self.shuttingDown():
-            try:
-                identifier, group = groups.popitem()
-            except:
-                break
-
-            # Save to DB
-            if group['library']:
-
-                # Add release
-                fireEvent('release.add', group = group)
-                library_item = fireEvent('library.update', identifier = group['library'].get('identifier'), single = True)
-                if library_item:
-                    added_identifier.append(library_item['identifier'])
-
-        return added_identifier
-
-
-    def scan(self, folder = None, files = [], simple = False, newer_than = 0):
+    def scan(self, folder = None, files = None, simple = False, newer_than = 0, on_found = None):
 
         folder = ss(os.path.normpath(folder))
 
@@ -141,7 +99,8 @@ class Scanner(Plugin):
         leftovers = []
 
         # Scan all files of the folder if no files are set
-        if len(files) == 0:
+        if not files:
+            check_file_date = True
             try:
                 files = []
                 for root, dirs, walk_files in os.walk(folder):
@@ -150,6 +109,7 @@ class Scanner(Plugin):
             except:
                 log.error('Failed getting files from %s: %s', (folder, traceback.format_exc()))
         else:
+            check_file_date = False
             files = [ss(x) for x in files]
 
         db = get_session()
@@ -279,8 +239,8 @@ class Scanner(Plugin):
                 del path_identifiers[identifier]
         del delete_identifiers
 
-        # Determine file types
-        processed_movies = {}
+        # Make sure we remove older / still extracting files
+        valid_files = {}
         while True and not self.shuttingDown():
             try:
                 identifier, group = movie_files.popitem()
@@ -302,7 +262,7 @@ class Scanner(Plugin):
                 if file_too_new:
                     break
 
-            if file_too_new:
+            if check_file_date and file_too_new:
                 try:
                     time_string = time.ctime(file_time[0])
                 except:
@@ -335,6 +295,19 @@ class Scanner(Plugin):
 
                     continue
 
+            valid_files[identifier] = group
+
+        del movie_files
+
+        # Determine file types
+        processed_movies = {}
+        total_found = len(valid_files)
+        while True and not self.shuttingDown():
+            try:
+                identifier, group = valid_files.popitem()
+            except:
+                break
+
             # Group extra (and easy) files first
             # images = self.getImages(group['unsorted_files'])
             group['files'] = {
@@ -395,9 +368,12 @@ class Scanner(Plugin):
                 movie = db.query(Movie).filter_by(library_id = group['library']['id']).first()
                 group['movie_id'] = None if not movie else movie.id
 
-
             processed_movies[identifier] = group
 
+            # Notify parent & progress on something found
+            if on_found:
+                on_found(group, total_found, total_found - len(processed_movies))
+
         if len(processed_movies) > 0:
             log.info('Found %s movies in the folder %s', (len(processed_movies), folder))
         else:

couchpotato/core/plugins/searcher/main.py

@@ -180,10 +180,10 @@ class Searcher(Plugin):
                             status_id = available_status.get('id')
                         )
                         db.add(rls)
-                        db.commit()
                     else:
-                        [db.delete(info) for info in rls.info]
-                        db.commit()
+                        [db.delete(old_info) for old_info in rls.info]
+
+                    db.commit()
 
                     for info in nzb:
                         try:
@@ -195,10 +195,11 @@ class Searcher(Plugin):
                                 value = toUnicode(nzb[info])
                             )
                             rls.info.append(rls_info)
-                            db.commit()
                         except InterfaceError:
                             log.debug('Couldn\'t add %s to ReleaseInfo: %s', (info, traceback.format_exc()))
 
+                    db.commit()
+
                     nzb['status_id'] = rls.status_id
 
 
@@ -245,38 +246,42 @@ class Searcher(Plugin):
 
         if successful:
 
-            # Mark release as snatched
-            db = get_session()
-            rls = db.query(Release).filter_by(identifier = md5(data['url'])).first()
-            rls.status_id = snatched_status.get('id')
-            db.commit()
+            try:
+                # Mark release as snatched
+                db = get_session()
+                rls = db.query(Release).filter_by(identifier = md5(data['url'])).first()
+                if rls:
+                    rls.status_id = snatched_status.get('id')
+                    db.commit()
 
-            log_movie = '%s (%s) in %s' % (getTitle(movie['library']), movie['library']['year'], rls.quality.label)
-            snatch_message = 'Snatched "%s": %s' % (data.get('name'), log_movie)
-            log.info(snatch_message)
-            fireEvent('movie.snatched', message = snatch_message, data = rls.to_dict())
+                    log_movie = '%s (%s) in %s' % (getTitle(movie['library']), movie['library']['year'], rls.quality.label)
+                    snatch_message = 'Snatched "%s": %s' % (data.get('name'), log_movie)
+                    log.info(snatch_message)
+                    fireEvent('movie.snatched', message = snatch_message, data = rls.to_dict())
 
+                # If renamer isn't used, mark movie done
+                if not Env.setting('enabled', 'renamer'):
+                    active_status = fireEvent('status.get', 'active', single = True)
+                    done_status = fireEvent('status.get', 'done', single = True)
+                    try:
+                        if movie['status_id'] == active_status.get('id'):
+                            for profile_type in movie['profile']['types']:
+                                if rls and profile_type['quality_id'] == rls.quality.id and profile_type['finish']:
+                                    log.info('Renamer disabled, marking movie as finished: %s', log_movie)
 
-            # If renamer isn't used, mark movie done
-            if not Env.setting('enabled', 'renamer'):
-                active_status = fireEvent('status.get', 'active', single = True)
-                done_status = fireEvent('status.get', 'done', single = True)
-                try:
-                    if movie['status_id'] == active_status.get('id'):
-                        for profile_type in movie['profile']['types']:
-                            if profile_type['quality_id'] == rls.quality.id and profile_type['finish']:
-                                log.info('Renamer disabled, marking movie as finished: %s', log_movie)
+                                    # Mark release done
+                                    rls.status_id = done_status.get('id')
+                                    db.commit()
 
-                                # Mark release done
-                                rls.status_id = done_status.get('id')
-                                db.commit()
+                                    # Mark movie done
+                                    mvie = db.query(Movie).filter_by(id = movie['id']).first()
+                                    mvie.status_id = done_status.get('id')
+                                    db.commit()
+                    except:
+                        log.error('Failed marking movie finished, renamer disabled: %s', traceback.format_exc())
 
-                                # Mark movie done
-                                mvie = db.query(Movie).filter_by(id = movie['id']).first()
-                                mvie.status_id = done_status.get('id')
-                                db.commit()
-                except Exception, e:
-                    log.error('Failed marking movie finished: %s %s', (e, traceback.format_exc()))
+            except:
+                log.error('Failed marking movie finished: %s', traceback.format_exc())
 
             return True
 
@@ -409,12 +414,17 @@ class Searcher(Plugin):
 
         return False
 
-    def correctYear(self, haystack, year, range):
+    def correctYear(self, haystack, year, year_range):
 
         for string in haystack:
-            if str(year) in string or str(int(year) + range) in string or str(int(year) - range) in string: # 1 year of is fine too
+
+            year_name = fireEvent('scanner.name_year', string, single = True)
+
+            if year_name and ((year - year_range) <= year_name.get('year') <= (year + year_range)):
+                log.debug('Movie year matches range: %s looking for %s', (year_name.get('year'), year))
                 return True
 
+        log.debug('Movie year doesn\'t matche range: %s looking for %s', (year_name.get('year'), year))
         return False
 
     def correctName(self, check_name, movie_name):

couchpotato/core/plugins/subtitle/main.py

@@ -1,6 +1,7 @@
 from couchpotato import get_session
 from couchpotato.core.event import addEvent, fireEvent
 from couchpotato.core.helpers.encoding import toUnicode
+from couchpotato.core.helpers.variable import splitString
 from couchpotato.core.logger import CPLog
 from couchpotato.core.plugins.base import Plugin
 from couchpotato.core.settings.model import Library, FileType
@@ -13,7 +14,7 @@ log = CPLog(__name__)
 
 class Subtitle(Plugin):
 
-    services = ['opensubtitles', 'thesubdb', 'subswiki']
+    services = ['opensubtitles', 'thesubdb', 'subswiki', 'podnapisi']
 
     def __init__(self):
         addEvent('renamer.before', self.searchSingle)
@@ -67,4 +68,4 @@ class Subtitle(Plugin):
         return False
 
     def getLanguages(self):
-        return [x.strip() for x in self.conf('languages').split(',')]
+        return splitString(self.conf('languages'))

couchpotato/core/plugins/trailer/main.py

@@ -12,7 +12,7 @@ class Trailer(Plugin):
     def __init__(self):
         addEvent('renamer.after', self.searchSingle)
 
-    def searchSingle(self, group):
+    def searchSingle(self, message = None, group = {}):
 
         if self.isDisabled() or len(group['files']['trailer']) > 0: return
 
@@ -28,6 +28,8 @@ class Trailer(Plugin):
             else:
                 log.debug('Trailer already exists: %s', destination)
 
+            group['renamed_files'].append(destination)
+
             # Download first and break
             break
 

couchpotato/core/providers/automation/base.py

@@ -28,9 +28,18 @@ class Automation(Plugin):
         return self.getIMDBids()
 
     def search(self, name, year = None, imdb_only = False):
+
+        prop_name = 'automation.cached.%s.%s' % (name, year)
+        cached_imdb = Env.prop(prop_name, default = False)
+        if cached_imdb and imdb_only:
+            return cached_imdb
+
         result = fireEvent('movie.search', q = '%s %s' % (name, year if year else ''), limit = 1, merge = True)
 
         if len(result) > 0:
+            if imdb_only and result[0].get('imdb'):
+                Env.prop(prop_name, result[0].get('imdb'))
+
             return result[0].get('imdb') if imdb_only else result[0]
         else:
             return None

couchpotato/core/providers/automation/bluray/main.py

@@ -2,7 +2,6 @@ from couchpotato.core.helpers.rss import RSS
 from couchpotato.core.helpers.variable import md5, tryInt
 from couchpotato.core.logger import CPLog
 from couchpotato.core.providers.automation.base import Automation
-from couchpotato.environment import Env
 import xml.etree.ElementTree as XMLTree
 
 log = CPLog(__name__)

couchpotato/core/providers/automation/imdb/main.py

@@ -2,9 +2,6 @@ from couchpotato.core.helpers.rss import RSS
 from couchpotato.core.helpers.variable import md5, getImdb
 from couchpotato.core.logger import CPLog
 from couchpotato.core.providers.automation.base import Automation
-from couchpotato.environment import Env
-from dateutil.parser import parse
-import time
 import traceback
 import xml.etree.ElementTree as XMLTree
 
@@ -34,10 +31,6 @@ class IMDB(Automation, RSS):
                 log.error('This isn\'t the correct url.: %s', rss_url)
                 continue
 
-            prop_name = 'automation.imdb.last_update.%s' % md5(rss_url)
-            last_update = float(Env.prop(prop_name, default = 0))
-
-            last_movie_added = 0
             try:
                 cache_key = 'imdb.rss.%s' % md5(rss_url)
 
@@ -46,20 +39,10 @@ class IMDB(Automation, RSS):
                 rss_movies = self.getElements(data, 'channel/item')
 
                 for movie in rss_movies:
-                    created = int(time.mktime(parse(self.getTextElement(movie, "pubDate")).timetuple()))
                     imdb = getImdb(self.getTextElement(movie, "link"))
-
-                    if created > last_movie_added:
-                        last_movie_added = created
-
-                    if not imdb or created <= last_update:
-                        continue
-
                     movies.append(imdb)
 
             except:
                 log.error('Failed loading IMDB watchlist: %s %s', (rss_url, traceback.format_exc()))
 
-            Env.prop(prop_name, last_movie_added)
-
         return movies

couchpotato/core/providers/automation/movies_io/main.py

@@ -3,10 +3,7 @@ from couchpotato.core.helpers.rss import RSS
 from couchpotato.core.helpers.variable import md5
 from couchpotato.core.logger import CPLog
 from couchpotato.core.providers.automation.base import Automation
-from couchpotato.environment import Env
-from dateutil.parser import parse
 from xml.etree.ElementTree import ParseError
-import time
 import traceback
 import xml.etree.ElementTree as XMLTree
 
@@ -33,10 +30,6 @@ class MoviesIO(Automation, RSS):
             if not enablers[index]:
                 continue
 
-            prop_name = 'automation.moviesio.last_update.%s' % md5(rss_url)
-            last_update = float(Env.prop(prop_name, default = 0))
-
-            last_movie_added = 0
             try:
                 cache_key = 'imdb.rss.%s' % md5(rss_url)
 
@@ -45,12 +38,6 @@ class MoviesIO(Automation, RSS):
                 rss_movies = self.getElements(data, 'channel/item')
 
                 for movie in rss_movies:
-                    created = int(time.mktime(parse(self.getTextElement(movie, "pubDate")).timetuple()))
-
-                    if created > last_movie_added:
-                        last_movie_added = created
-                    if created <= last_update:
-                        continue
 
                     nameyear = fireEvent('scanner.name_year', self.getTextElement(movie, "title"), single = True)
                     imdb = self.search(nameyear.get('name'), nameyear.get('year'), imdb_only = True)
@@ -64,6 +51,4 @@ class MoviesIO(Automation, RSS):
             except:
                 log.error('Failed loading Movies.io watchlist: %s %s', (rss_url, traceback.format_exc()))
 
-            Env.prop(prop_name, last_movie_added)
-
         return movies

couchpotato/core/providers/metadata/base.py

@@ -2,6 +2,7 @@ from couchpotato.core.event import addEvent, fireEvent
 from couchpotato.core.helpers.variable import mergeDicts
 from couchpotato.core.logger import CPLog
 from couchpotato.core.plugins.base import Plugin
+from couchpotato.environment import Env
 import os
 import shutil
 import traceback
@@ -16,23 +17,23 @@ class MetaDataBase(Plugin):
     def __init__(self):
         addEvent('renamer.after', self.create)
 
-    def create(self, release):
+    def create(self, message = None, group = {}):
         if self.isDisabled(): return
 
         log.info('Creating %s metadata.', self.getName())
 
         # Update library to get latest info
         try:
-            updated_library = fireEvent('library.update', release['library']['identifier'], force = True, single = True)
-            release['library'] = mergeDicts(release['library'], updated_library)
+            updated_library = fireEvent('library.update', group['library']['identifier'], force = True, single = True)
+            group['library'] = mergeDicts(group['library'], updated_library)
         except:
             log.error('Failed to update movie, before creating metadata: %s', traceback.format_exc())
 
-        root_name = self.getRootName(release)
+        root_name = self.getRootName(group)
         meta_name = os.path.basename(root_name)
         root = os.path.dirname(root_name)
 
-        movie_info = release['library'].get('info')
+        movie_info = group['library'].get('info')
 
         for file_type in ['nfo', 'thumbnail', 'fanart']:
             try:
@@ -42,13 +43,19 @@ class MetaDataBase(Plugin):
                 if name and self.conf('meta_' + file_type):
 
                     # Get file content
-                    content = getattr(self, 'get' + file_type.capitalize())(movie_info = movie_info, data = release)
+                    content = getattr(self, 'get' + file_type.capitalize())(movie_info = movie_info, data = group)
                     if content:
                         log.debug('Creating %s file: %s', (file_type, name))
                         if os.path.isfile(content):
                             shutil.copy2(content, name)
                         else:
                             self.createFile(name, content)
+                            group['renamed_files'].append(name)
+
+                        try:
+                            os.chmod(name, Env.getPermission('file'))
+                        except:
+                            log.debug('Failed setting permissions for %s: %s', (name, traceback.format_exc()))
 
             except:
                 log.error('Unable to create %s file: %s', (file_type, traceback.format_exc()))

couchpotato/core/providers/movie/imdbapi/main.py

@@ -1,6 +1,6 @@
 from couchpotato.core.event import addEvent, fireEvent
 from couchpotato.core.helpers.encoding import tryUrlencode
-from couchpotato.core.helpers.variable import tryInt, tryFloat
+from couchpotato.core.helpers.variable import tryInt, tryFloat, splitString
 from couchpotato.core.logger import CPLog
 from couchpotato.core.providers.movie.base import MovieProvider
 import json
@@ -27,8 +27,10 @@ class IMDBAPI(MovieProvider):
 
         name_year = fireEvent('scanner.name_year', q, single = True)
 
-        if not q or not name_year or (name_year and not name_year.get('name')):
-            return []
+        if not name_year or (name_year and not name_year.get('name')):
+            name_year = {
+                'name': q
+            }
 
         cache_key = 'imdbapi.cache.%s' % q
         cached = self.getCache(cache_key, self.urls['search'] % tryUrlencode({'t': name_year.get('name'), 'y': name_year.get('year', '')}), timeout = 3)
@@ -97,10 +99,10 @@ class IMDBAPI(MovieProvider):
                 'released': movie.get('Released', ''),
                 'year': year if isinstance(year, (int)) else None,
                 'plot': movie.get('Plot', ''),
-                'genres': movie.get('Genre', '').split(','),
-                'directors': movie.get('Director', '').split(','),
-                'writers': movie.get('Writer', '').split(','),
-                'actors': movie.get('Actors', '').split(','),
+                'genres': splitString(movie.get('Genre', '')),
+                'directors': splitString(movie.get('Director', '')),
+                'writers': splitString(movie.get('Writer', '')),
+                'actors': splitString(movie.get('Actors', '')),
             }
         except:
             log.error('Failed parsing IMDB API json: %s', traceback.format_exc())

couchpotato/core/providers/nzb/mysterbin/__init__.py

@@ -10,7 +10,7 @@ config = [{
             'tab': 'searcher',
             'subtab': 'nzb_providers',
             'name': 'Mysterbin',
-            'description': 'Free provider, less accurate. See <a href="http://www.mysterbin.com/">Mysterbin</a>',
+            'description': 'Free provider, less accurate. See <a href="https://www.mysterbin.com/">Mysterbin</a>',
             'options': [
                 {
                     'name': 'enabled',

couchpotato/core/providers/nzb/newznab/__init__.py

@@ -11,7 +11,7 @@ config = [{
             'subtab': 'nzb_providers',
             'name': 'newznab',
             'order': 10,
-            'description': 'Enable multiple NewzNab providers such as <a href="http://nzb.su" target="_blank">NZB.su</a> and <a href="http://nzbs.org" target="_blank">nzbs.org</a>',
+            'description': 'Enable multiple NewzNab providers such as <a href="https://nzb.su" target="_blank">NZB.su</a> and <a href="https://nzbs.org" target="_blank">nzbs.org</a>',
             'wizard': True,
             'options': [
                 {

couchpotato/core/providers/nzb/newznab/main.py

@@ -1,7 +1,7 @@
 from couchpotato.core.event import fireEvent
 from couchpotato.core.helpers.encoding import tryUrlencode
 from couchpotato.core.helpers.rss import RSS
-from couchpotato.core.helpers.variable import cleanHost
+from couchpotato.core.helpers.variable import cleanHost, splitString
 from couchpotato.core.logger import CPLog
 from couchpotato.core.providers.nzb.base import NZBProvider
 from couchpotato.environment import Env
@@ -161,9 +161,9 @@ class Newznab(NZBProvider, RSS):
 
     def getHosts(self):
 
-        uses = [x.strip() for x in str(self.conf('use')).split(',')]
-        hosts = [x.strip() for x in self.conf('host').split(',')]
-        api_keys = [x.strip() for x in self.conf('api_key').split(',')]
+        uses = splitString(str(self.conf('use')))
+        hosts = splitString(self.conf('host'))
+        api_keys = splitString(self.conf('api_key'))
 
         list = []
         for nr in range(len(hosts)):

couchpotato/core/providers/nzb/nzbindex/__init__.py

@@ -10,7 +10,7 @@ config = [{
             'tab': 'searcher',
             'subtab': 'nzb_providers',
             'name': 'nzbindex',
-            'description': 'Free provider, less accurate. See <a href="http://www.nzbindex.nl/">NZBIndex</a>',
+            'description': 'Free provider, less accurate. See <a href="https://www.nzbindex.com/">NZBIndex</a>',
             'options': [
                 {
                     'name': 'enabled',

couchpotato/core/providers/nzb/nzbindex/main.py

@@ -19,8 +19,8 @@ log = CPLog(__name__)
 class NzbIndex(NZBProvider, RSS):
 
     urls = {
-        'download': 'http://www.nzbindex.nl/download/',
-        'api': 'http://www.nzbindex.nl/rss/',
+        'download': 'https://www.nzbindex.com/download/',
+        'api': 'https://www.nzbindex.com/rss/',
     }
 
     http_time_between_calls = 1 # Seconds

couchpotato/core/providers/torrent/kickasstorrents/main.py

@@ -1,6 +1,7 @@
 from bs4 import BeautifulSoup
 from couchpotato.core.event import fireEvent
-from couchpotato.core.helpers.variable import tryInt
+from couchpotato.core.helpers.encoding import simplifyString
+from couchpotato.core.helpers.variable import tryInt, getTitle
 from couchpotato.core.logger import CPLog
 from couchpotato.core.providers.torrent.base import TorrentProvider
 import re
@@ -12,9 +13,9 @@ log = CPLog(__name__)
 class KickAssTorrents(TorrentProvider):
 
     urls = {
-        'test': 'http://kat.ph/',
-        'detail': 'http://kat.ph/%s',
-        'search': 'http://kat.ph/i%s/',
+        'test': 'https://kat.ph/',
+        'detail': 'https://kat.ph/%s',
+        'search': 'https://kat.ph/%s-i%s/',
     }
 
     cat_ids = [
@@ -35,8 +36,10 @@ class KickAssTorrents(TorrentProvider):
         if self.isDisabled():
             return results
 
+        title = simplifyString(getTitle(movie['library'])).replace(' ', '-')
+
         cache_key = 'kickasstorrents.%s.%s' % (movie['library']['identifier'], quality.get('identifier'))
-        data = self.getCache(cache_key, self.urls['search'] % (movie['library']['identifier'].replace('tt', '')))
+        data = self.getCache(cache_key, self.urls['search'] % (title, movie['library']['identifier'].replace('tt', '')))
         if data:
 
             cat_ids = self.getCatId(quality['identifier'])

couchpotato/core/providers/torrent/passthepopcorn/__init__.py

@@ -10,7 +10,7 @@ config = [{
             'tab': 'searcher',
             'subtab': 'torrent_providers',
             'name': 'PassThePopcorn',
-            'description': 'See <a href="http://passthepopcorn.me">PassThePopcorn.me</a>',
+            'description': 'See <a href="https://passthepopcorn.me">PassThePopcorn.me</a>',
             'options': [
                 {
                     'name': 'enabled',

couchpotato/core/providers/torrent/publichd/__init__.py

@@ -10,7 +10,7 @@ config = [{
             'tab': 'searcher',
             'subtab': 'torrent_providers',
             'name': 'PublicHD',
-            'description': 'Public Torrent site with only HD content. See <a href="http://publichd.eu/">PublicHD</a>',
+            'description': 'Public Torrent site with only HD content. See <a href="https://publichd.eu/">PublicHD</a>',
             'options': [
                 {
                     'name': 'enabled',

couchpotato/core/providers/torrent/publichd/main.py

@@ -14,9 +14,9 @@ log = CPLog(__name__)
 class PublicHD(TorrentProvider):
 
     urls = {
-        'test': 'http://publichd.eu',
-        'detail': 'http://publichd.eu/index.php?page=torrent-details&id=%s',
-        'search': 'http://publichd.eu/index.php',
+        'test': 'https://publichd.eu',
+        'detail': 'https://publichd.eu/index.php?page=torrent-details&id=%s',
+        'search': 'https://publichd.eu/index.php',
     }
     http_time_between_calls = 0
 

couchpotato/core/providers/torrent/scenehd/__init__.py

@@ -10,7 +10,7 @@ config = [{
             'tab': 'searcher',
             'subtab': 'torrent_providers',
             'name': 'SceneHD',
-            'description': 'See <a href="http://scenehd.org">SceneHD</a>',
+            'description': 'See <a href="https://scenehd.org">SceneHD</a>',
             'options': [
                 {
                     'name': 'enabled',

couchpotato/core/providers/torrent/scenehd/main.py

@@ -12,11 +12,11 @@ log = CPLog(__name__)
 class SceneHD(TorrentProvider):
 
     urls = {
-        'test': 'http://scenehd.org/',
-        'login' : 'http://scenehd.org/takelogin.php',
-        'detail': 'http://scenehd.org/details.php?id=%s',
-        'search': 'http://scenehd.org/browse.php?ajax',
-        'download': 'http://scenehd.org/download.php?id=%s',
+        'test': 'https://scenehd.org/',
+        'login' : 'https://scenehd.org/takelogin.php',
+        'detail': 'https://scenehd.org/details.php?id=%s',
+        'search': 'https://scenehd.org/browse.php?ajax',
+        'download': 'https://scenehd.org/download.php?id=%s',
     }
 
     http_time_between_calls = 1 #seconds

couchpotato/core/providers/trailer/hdtrailers/main.py

@@ -4,6 +4,7 @@ from couchpotato.core.helpers.variable import mergeDicts, getTitle
 from couchpotato.core.logger import CPLog
 from couchpotato.core.providers.trailer.base import TrailerProvider
 from string import digits, ascii_letters
+from urllib2 import HTTPError
 import re
 
 log = CPLog(__name__)
@@ -22,7 +23,12 @@ class HDTrailers(TrailerProvider):
         movie_name = getTitle(group['library'])
 
         url = self.urls['api'] % self.movieUrlName(movie_name)
-        data = self.getCache('hdtrailers.%s' % group['library']['identifier'], url)
+        try:
+            data = self.getCache('hdtrailers.%s' % group['library']['identifier'], url, show_error = False)
+        except HTTPError:
+            log.debug('No page found for: %s', movie_name)
+            data = None
+
         result_data = {'480p':[], '720p':[], '1080p':[]}
 
         if not data:
@@ -47,7 +53,14 @@ class HDTrailers(TrailerProvider):
         movie_name = getTitle(group['library'])
 
         url = "%s?%s" % (self.urls['backup'], tryUrlencode({'s':movie_name}))
-        data = self.getCache('hdtrailers.alt.%s' % group['library']['identifier'], url)
+        try:
+            data = self.getCache('hdtrailers.alt.%s' % group['library']['identifier'], url, show_error = False)
+        except HTTPError:
+            log.debug('No alternative page found for: %s', movie_name)
+            data = None
+
+        if not data:
+            return results
 
         try:
             tables = SoupStrainer('div')

couchpotato/static/scripts/library/prefix_free.js

@@ -1,5 +1,5 @@
 /**
- * StyleFix 1.0.3
+ * StyleFix 1.0.3 & PrefixFree 1.0.7
  * @author Lea Verou
  * MIT license
  */
@@ -122,9 +122,9 @@ var self = window.StyleFix = {
 			.splice(index === undefined? self.fixers.length : index, 0, fixer);
 	},
 	
-	fix: function(css, raw) {
+	fix: function(css, raw, element) {
 		for(var i=0; i<self.fixers.length; i++) {
-			css = self.fixers[i](css, raw) || css;
+			css = self.fixers[i](css, raw, element) || css;
 		}
 		
 		return css;
@@ -157,11 +157,9 @@ function $(expr, con) {
 })();
 
 /**
- * PrefixFree 1.0.6
- * @author Lea Verou
- * MIT license
+ * PrefixFree
  */
-(function(root, undefined){
+(function(root){
 
 if(!window.StyleFix || !window.getComputedStyle) {
 	return;
@@ -181,9 +179,17 @@ function fix(what, before, after, replacement, css) {
 }
 
 var self = window.PrefixFree = {
-	prefixCSS: function(css, raw) {
+	prefixCSS: function(css, raw, element) {
 		var prefix = self.prefix;
 		
+		// Gradient angles hotfix
+		if(self.functions.indexOf('linear-gradient') > -1) {
+			// Gradients are supported with a prefix, convert angles to legacy
+			css = css.replace(/(\s|:|,)(repeating-)?linear-gradient\(\s*(-?\d*\.?\d*)deg/ig, function ($0, delim, repeating, deg) {
+				return delim + (repeating || '') + 'linear-gradient(' + (90-deg) + 'deg';
+			});
+		}
+		
 		css = fix('functions', '(\\s|:|,)', '\\s*\\(', '$1' + prefix + '$2(', css);
 		css = fix('keywords', '(\\s|:)', '(\\s|;|\\}|$)', '$1' + prefix + '$2$3', css);
 		css = fix('properties', '(^|\\{|\\s|;)', '\\s*:', '$1' + prefix + '$2:', css);
@@ -205,6 +211,9 @@ var self = window.PrefixFree = {
 		// Fix double prefixing
 		css = css.replace(RegExp('-' + prefix, 'g'), '-');
 		
+		// Prefix wildcard
+		css = css.replace(/-\*-(?=[a-z]+)/gi, self.prefix);
+		
 		return css;
 	},
 	

couchpotato/static/scripts/page/manage.js

@@ -29,9 +29,35 @@ Page.Manage = new Class({
 				'identifier': 'manage',
 				'status': 'done',
 				'actions': MovieActions,
-				'menu': [self.refresh_button, self.refresh_quick]
+				'menu': [self.refresh_button, self.refresh_quick],
+				'on_empty_element': new Element('div.empty_manage').adopt(
+					new Element('div', {
+						'text': 'Seems like you don\'t have anything in your library yet.'
+					}),
+					new Element('div', {
+						'text': 'Add your existing movie folders in '
+					}).adopt(
+						new Element('a', {
+							'text': 'Settings > Manage',
+							'href': App.createUrl('settings/manage')
+						})
+					),
+					new Element('div.after_manage', {
+						'text': 'When you\'ve done that, hit this button → '
+					}).adopt(
+						new Element('a.button.green', {
+							'text': 'Hit me, but not to hard',
+							'events':{
+								'click': self.refresh.bind(self, true)
+							}
+						})
+					)
+				)
 			});
 			$(self.list).inject(self.el);
+
+			// Check if search is in progress
+			self.startProgressInterval();
 		}
 
 	},
@@ -39,11 +65,55 @@ Page.Manage = new Class({
 	refresh: function(full){
 		var self = this;
 
-		Api.request('manage.update', {
-			'data': {
-				'full': +full
-			}
-		})
+		if(!self.update_in_progress){
+
+			Api.request('manage.update', {
+				'data': {
+					'full': +full
+				}
+			})
+
+			self.startProgressInterval();
+
+		}
+
+	},
+
+	startProgressInterval: function(){
+		var self = this;
+
+		self.progress_interval = setInterval(function(){
+
+			Api.request('manage.progress', {
+				'onComplete': function(json){
+					self.update_in_progress = true;
+
+					if(!json.progress){
+						clearInterval(self.progress_interval);
+						self.update_in_progress = false;
+						if(self.progress_container){
+							self.progress_container.destroy();
+							self.list.update();
+						}
+					}
+					else {
+						if(!self.progress_container)
+							self.progress_container = new Element('div.progress').inject(self.list.navigation, 'after')
+
+						self.progress_container.empty();
+
+						Object.each(json.progress, function(progress, folder){
+							new Element('div').adopt(
+								new Element('span.folder', {'text': folder}),
+								new Element('span.percentage', {'text': progress.total ? (((progress.total-progress.to_go)/progress.total)*100).round() + '%' : '0%'})
+							).inject(self.progress_container)
+						});
+
+					}
+				}
+			})
+
+		}, 1000);
 
 	}
 

couchpotato/static/scripts/page/wanted.js

@@ -18,9 +18,6 @@ Page.Wanted = new Class({
 				}
 			});
 
-			// See if userscript can be installed
-
-
 			// Wanted movies
 			self.wanted = new MovieList({
 				'identifier': 'wanted',
@@ -28,13 +25,7 @@ Page.Wanted = new Class({
 				'actions': MovieActions,
 				'add_new': true,
 				'menu': [self.manual_search],
-				'on_empty_element': App.createUserscriptButtons().setStyles({
-					'background-image': "url('"+Api.createUrl('static/images/emptylist.png')+"')",
-					'height': 750,
-					'width': 800,
-					'padding-top': 260,
-					'margin-top': -50
-				})
+				'on_empty_element': App.createUserscriptButtons().addClass('empty_wanted')
 			});
 			$(self.wanted).inject(self.el);
 
@@ -145,7 +136,7 @@ window.addEvent('domready', function(){
 							'text': profile.label ? profile.label : profile.data.label
 						}).inject(self.profile_select);
 
-						if(self.movie.profile && self.movie.profile.data.id == profile_id)
+						if(self.movie.profile && self.movie.profile.data && self.movie.profile.data.id == profile_id)
 							self.profile_select.set('value', profile_id);
 					});
 

couchpotato/static/style/main.css

@@ -271,7 +271,7 @@ body > .spinner, .mask{
 		box-shadow: inset 0 1px 0 rgba(255,255,255,.6), 0 0 3px rgba(0,0,0,.7);
 		background-color: #1b79b8;
 		text-shadow: none;
-		background-image: linear-gradient(0deg, rgba(255,255,255,.3) 0%, rgba(255,255,255,.1) 100%);
+		background-image: -*-linear-gradient(0deg, rgba(255,255,255,.3) 0%, rgba(255,255,255,.1) 100%);
 	}
 
 	.header .notification_menu .wrapper {
@@ -355,7 +355,7 @@ body > .spinner, .mask{
 		border-radius:30px;
 
 		box-shadow: 0 1px 1px rgba(0,0,0,0.35), inset 0 1px 0px rgba(255,255,255,0.20);
-		background: url('../images/sprite.png') no-repeat 94% -53px, linear-gradient(
+		background: url('../images/sprite.png') no-repeat 94% -53px, -*-linear-gradient(
 		    270deg,
 		    #5b9bd1 0%,
 		    #406db8 100%
@@ -431,7 +431,7 @@ body > .spinner, .mask{
 	border: 1px solid #252930;
 	box-shadow: inset 0 1px 0px rgba(255,255,255,0.20), 0 0 3px rgba(0,0,0, 0.2);
 	background: rgb(55,62,74);
-	background-image: linear-gradient(
+	background-image: -*-linear-gradient(
 	    90deg,
 	    rgb(55,62,74) 0%,
 	    rgb(73,83,98) 100%
@@ -520,7 +520,7 @@ body > .spinner, .mask{
 		text-align: center;
 		color: #000;
 		text-shadow: none;
-		background-image: linear-gradient(
+		background-image: -*-linear-gradient(
 		    45deg,
 		    rgb(200,200,200) 0%,
 		    rgb(255,255,255) 100%
@@ -592,7 +592,7 @@ body > .spinner, .mask{
 		overflow: hidden;
 		transition: all .6s cubic-bezier(0.9,0,0.1,1);
 		box-shadow: 0 1px 1px rgba(0,0,0,0.35), inset 0 1px 0px rgba(255,255,255,0.20);
-		background-image: linear-gradient(
+		background-image: -*-linear-gradient(
 		    270deg,
 		    #5b9bd1 0%,
 		    #406db8 100%

couchpotato/static/style/page/settings.css

@@ -16,7 +16,7 @@
 		padding: 40px 0;
 		margin: 0;
 		min-height: 470px;
-		background-image: linear-gradient(
+		background-image: -*-linear-gradient(
 		    20deg,
 		    rgba(0,0,0,0) 50%,
 		    rgba(0,0,0,0.3) 100%
@@ -365,7 +365,7 @@
 				border-radius: 2px;
 			}
 			.page .tag_input > ul:hover > li.choice {
-				background: linear-gradient(
+				background: -*-linear-gradient(
 				    270deg,
 				    rgba(255,255,255,0.3) 0%,
 				    rgba(255,255,255,0.1) 100%
@@ -373,7 +373,7 @@
 			}
 			.page .tag_input > ul > li.choice:hover,
 			.page .tag_input > ul > li.choice.selected {
-				background: linear-gradient(
+				background: -*-linear-gradient(
 				    270deg,
 				    #5b9bd1 0%,
 				    #406db8 100%
@@ -412,7 +412,7 @@
 				margin: -9px 0 0 -16px;
 				border-radius: 30px 30px 0 0;
 				cursor: pointer;
-				background: url('../../images/icon.delete.png') no-repeat center 2px, linear-gradient(
+				background: url('../../images/icon.delete.png') no-repeat center 2px, -*-linear-gradient(
 				   270deg,
 				    #5b9bd1 0%,
 				    #5b9bd1 100%

couchpotato/templates/_desktop.html

@@ -6,7 +6,6 @@
 		<link rel="stylesheet" href="{{ url_for('web.static', filename='style/uniform.css') }}" type="text/css">
 
 		<link rel="stylesheet" href="{{ url_for('web.static', filename='style/page/settings.css') }}" type="text/css">
-		<link rel="stylesheet" href="{{ url_for('web.static', filename='style/page/wanted.css') }}" type="text/css">
 
 		<script type="text/javascript" src="{{ url_for('web.static', filename='scripts/library/mootools.js') }}"></script>
 		<script type="text/javascript" src="{{ url_for('web.static', filename='scripts/library/mootools_more.js') }}"></script>

libs/guessit/__init__.py

@@ -18,12 +18,58 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
-__version__ = '0.4.2'
+
+__version__ = '0.5.2'
 __all__ = ['Guess', 'Language',
            'guess_file_info', 'guess_video_info',
            'guess_movie_info', 'guess_episode_info']
 
 
+# Do python3 detection before importing any other module, to be sure that
+# it will then always be available
+# with code from http://lucumr.pocoo.org/2011/1/22/forwards-compatible-python/
+import sys
+if sys.version_info[0] >= 3:
+    PY3 = True
+    unicode_text_type = str
+    native_text_type = str
+    base_text_type = str
+    def u(x):
+        return str(x)
+    def s(x):
+        return x
+    class UnicodeMixin(object):
+        __str__ = lambda x: x.__unicode__()
+    import binascii
+    def to_hex(x):
+        return binascii.hexlify(x).decode('utf-8')
+
+else:
+    PY3 = False
+    __all__ = [ str(s) for s in __all__ ] # fix imports for python2
+    unicode_text_type = unicode
+    native_text_type = str
+    base_text_type = basestring
+    def u(x):
+        if isinstance(x, str):
+            return x.decode('utf-8')
+        return unicode(x)
+    def s(x):
+        if isinstance(x, unicode):
+            return x.encode('utf-8')
+        if isinstance(x, list):
+            return [ s(y) for y in x ]
+        if isinstance(x, tuple):
+            return tuple(s(y) for y in x)
+        if isinstance(x, dict):
+            return dict((s(key), s(value)) for key, value in x.items())
+        return x
+    class UnicodeMixin(object):
+        __str__ = lambda x: unicode(x).encode('utf-8')
+    def to_hex(x):
+        return x.encode('hex')
+
+
 from guessit.guess import Guess, merge_all
 from guessit.language import Language
 from guessit.matcher import IterativeMatcher
@@ -32,6 +78,7 @@ import logging
 log = logging.getLogger(__name__)
 
 
+
 class NullHandler(logging.Handler):
     def emit(self, record):
         pass
@@ -45,7 +92,7 @@ def guess_file_info(filename, filetype, info=None):
     """info can contain the names of the various plugins, such as 'filename' to
     detect filename info, or 'hash_md5' to get the md5 hash of the file.
 
-    >>> guess_file_info('test/dummy.srt', 'autodetect', info = ['hash_md5', 'hash_sha1'])
+    >>> guess_file_info('tests/dummy.srt', 'autodetect', info = ['hash_md5', 'hash_sha1'])
     {'hash_md5': 'e781de9b94ba2753a8e2945b2c0a123d', 'hash_sha1': 'bfd18e2f4e5d59775c2bc14d80f56971891ed620'}
     """
     result = []
@@ -54,7 +101,7 @@ def guess_file_info(filename, filetype, info=None):
     if info is None:
         info = ['filename']
 
-    if isinstance(info, basestring):
+    if isinstance(info, base_text_type):
         info = [info]
 
     for infotype in info:
@@ -67,7 +114,7 @@ def guess_file_info(filename, filetype, info=None):
             try:
                 result.append(Guess({'hash_mpc': hash_file(filename)},
                                     confidence=1.0))
-            except Exception, e:
+            except Exception as e:
                 log.warning('Could not compute MPC-style hash because: %s' % e)
 
         elif infotype == 'hash_ed2k':
@@ -75,7 +122,7 @@ def guess_file_info(filename, filetype, info=None):
             try:
                 result.append(Guess({'hash_ed2k': hash_file(filename)},
                                     confidence=1.0))
-            except Exception, e:
+            except Exception as e:
                 log.warning('Could not compute ed2k hash because: %s' % e)
 
         elif infotype.startswith('hash_'):
@@ -97,17 +144,28 @@ def guess_file_info(filename, filetype, info=None):
             hasherobjs = dict(hashers).values()
 
             with open(filename, 'rb') as f:
-                for chunk in iter(lambda: f.read(blocksize), ''):
+                chunk = f.read(blocksize)
+                while chunk:
                     for hasher in hasherobjs:
                         hasher.update(chunk)
+                    chunk = f.read(blocksize)
 
             for infotype, hasher in hashers:
                 result.append(Guess({infotype: hasher.hexdigest()},
                                     confidence=1.0))
-        except Exception, e:
+        except Exception as e:
             log.warning('Could not compute hash because: %s' % e)
 
-    return merge_all(result)
+    result = merge_all(result)
+
+    # last minute adjustments
+
+    # if country is in the guessed properties, make it part of the filename
+    if 'country' in result:
+        result['series'] += ' (%s)' % result['country'].alpha2.upper()
+
+
+    return result
 
 
 def guess_video_info(filename, info=None):

libs/guessit/country.py

@@ -19,10 +19,12 @@
 #
 
 from __future__ import unicode_literals
-from guessit import fileutils
-from guessit.textutils import to_unicode
+from guessit import UnicodeMixin, base_text_type, u
+from guessit.fileutils import load_file_in_same_dir
 import logging
 
+__all__ = [ 'Country' ]
+
 log = logging.getLogger(__name__)
 
 
@@ -32,8 +34,7 @@ log = logging.getLogger(__name__)
 # "An English name, an alpha-2 code (when given),
 # an alpha-3 code (when given), a numeric code, and an ISO 31666-2 code
 # are all separated by pipe (|) characters."
-_iso3166_contents = fileutils.load_file_in_same_dir(__file__,
-                                                    'ISO-3166-1_utf8.txt').decode('utf-8')
+_iso3166_contents = load_file_in_same_dir(__file__, 'ISO-3166-1_utf8.txt')
 
 country_matrix = [ l.strip().split('|')
                    for l in _iso3166_contents.strip().split('\n') ]
@@ -59,7 +60,7 @@ country_alpha3_to_alpha2 = dict((c[2].lower(), c[1].lower()) for c in country_ma
 
 
 
-class Country(object):
+class Country(UnicodeMixin):
     """This class represents a country.
 
     You can initialize it with pretty much anything, as it knows conversion
@@ -67,7 +68,7 @@ class Country(object):
     """
 
     def __init__(self, country, strict=False):
-        country = to_unicode(country.strip().lower())
+        country = u(country.strip().lower())
         self.alpha3 = country_to_alpha3.get(country)
 
         if self.alpha3 is None and strict:
@@ -93,7 +94,7 @@ class Country(object):
         if isinstance(other, Country):
             return self.alpha3 == other.alpha3
 
-        if isinstance(other, basestring):
+        if isinstance(other, base_text_type):
             try:
                 return self == Country(other)
             except ValueError:
@@ -107,9 +108,5 @@ class Country(object):
     def __unicode__(self):
         return self.english_name
 
-    def __str__(self):
-        return unicode(self).encode('utf-8')
-
     def __repr__(self):
         return 'Country(%s)' % self.english_name
-

libs/guessit/date.py

@@ -18,9 +18,12 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
+from __future__ import unicode_literals
 import datetime
 import re
 
+def valid_year(year):
+    return 1920 < year < datetime.date.today().year + 5
 
 def search_year(string):
     """Looks for year patterns, and if found return the year and group span.
@@ -40,7 +43,7 @@ def search_year(string):
     match = re.search(r'[^0-9]([0-9]{4})[^0-9]', string)
     if match:
         year = int(match.group(1))
-        if 1920 < year < datetime.date.today().year + 5:
+        if valid_year(year):
             return (year, match.span(1))
 
     return (None, None)

libs/guessit/fileutils.py

@@ -18,6 +18,8 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
+from __future__ import unicode_literals
+from guessit import s, u
 import os.path
 import zipfile
 
@@ -31,10 +33,10 @@ def split_path(path):
      - the drive letter on Windows systems (eg: r'C:\')
      - the mount point '\\' on Windows systems (eg: r'\\host\share')
 
-    >>> split_path('/usr/bin/smewt')
+    >>> s(split_path('/usr/bin/smewt'))
     ['/', 'usr', 'bin', 'smewt']
 
-    >>> split_path('relative_path/to/my_folder/')
+    >>> s(split_path('relative_path/to/my_folder/'))
     ['relative_path', 'to', 'my_folder']
 
     """
@@ -65,7 +67,7 @@ def split_path(path):
 def file_in_same_dir(ref_file, desired_file):
     """Return the path for a file in the same dir as a given reference file.
 
-    >>> file_in_same_dir('~/smewt/smewt.db', 'smewt.settings')
+    >>> s(file_in_same_dir('~/smewt/smewt.db', 'smewt.settings'))
     '~/smewt/smewt.settings'
 
     """
@@ -82,4 +84,4 @@ def load_file_in_same_dir(ref_file, filename):
             zfile = zipfile.ZipFile(zfilename)
             return zfile.read('/'.join(path[i + 1:]))
 
-    return open(os.path.join(*path)).read()
+    return u(open(os.path.join(*path)).read())

libs/guessit/guess.py

@@ -18,6 +18,10 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
+from __future__ import unicode_literals
+from guessit import UnicodeMixin, s, u, base_text_type
+from guessit.language import Language
+from guessit.country import Country
 import json
 import datetime
 import logging
@@ -25,7 +29,7 @@ import logging
 log = logging.getLogger(__name__)
 
 
-class Guess(dict):
+class Guess(UnicodeMixin, dict):
     """A Guess is a dictionary which has an associated confidence for each of
     its values.
 
@@ -44,23 +48,21 @@ class Guess(dict):
         for prop in self:
             self._confidence[prop] = confidence
 
-    def to_utf8_dict(self):
-        from guessit.language import Language
+
+    def to_dict(self):
         data = dict(self)
         for prop, value in data.items():
             if isinstance(value, datetime.date):
                 data[prop] = value.isoformat()
-            elif isinstance(value, Language):
-                data[prop] = str(value)
-            elif isinstance(value, unicode):
-                data[prop] = value.encode('utf-8')
+            elif isinstance(value, (Language, Country, base_text_type)):
+                data[prop] = u(value)
             elif isinstance(value, list):
-                data[prop] = [str(x) for x in value]
+                data[prop] = [u(x) for x in value]
 
         return data
 
     def nice_string(self):
-        data = self.to_utf8_dict()
+        data = self.to_dict()
 
         parts = json.dumps(data, indent=4).split('\n')
         for i, p in enumerate(parts):
@@ -72,8 +74,8 @@ class Guess(dict):
 
         return '\n'.join(parts)
 
-    def __str__(self):
-        return str(self.to_utf8_dict())
+    def __unicode__(self):
+        return u(self.to_dict())
 
     def confidence(self, prop):
         return self._confidence.get(prop, -1)
@@ -138,16 +140,16 @@ def choose_string(g1, g2):
     differ very little, such as one string being the other one with the 'the' word
     prepended to it.
 
-    >>> choose_string(('Hello', 0.75), ('World', 0.5))
+    >>> s(choose_string(('Hello', 0.75), ('World', 0.5)))
     ('Hello', 0.25)
 
-    >>> choose_string(('Hello', 0.5), ('hello', 0.5))
+    >>> s(choose_string(('Hello', 0.5), ('hello', 0.5)))
     ('Hello', 0.75)
 
-    >>> choose_string(('Hello', 0.4), ('Hello World', 0.4))
+    >>> s(choose_string(('Hello', 0.4), ('Hello World', 0.4)))
     ('Hello', 0.64)
 
-    >>> choose_string(('simpsons', 0.5), ('The Simpsons', 0.5))
+    >>> s(choose_string(('simpsons', 0.5), ('The Simpsons', 0.5)))
     ('The Simpsons', 0.75)
 
     """
@@ -285,12 +287,12 @@ def merge_all(guesses, append=None):
     You can specify a list of properties that should be appended into a list
     instead of being merged.
 
-    >>> merge_all([ Guess({ 'season': 2 }, confidence = 0.6),
-    ...             Guess({ 'episodeNumber': 13 }, confidence = 0.8) ])
+    >>> s(merge_all([ Guess({ 'season': 2 }, confidence = 0.6),
+    ...               Guess({ 'episodeNumber': 13 }, confidence = 0.8) ]))
     {'season': 2, 'episodeNumber': 13}
 
-    >>> merge_all([ Guess({ 'episodeNumber': 27 }, confidence = 0.02),
-    ...             Guess({ 'season': 1 }, confidence = 0.2) ])
+    >>> s(merge_all([ Guess({ 'episodeNumber': 27 }, confidence = 0.02),
+    ...               Guess({ 'season': 1 }, confidence = 0.2) ]))
     {'season': 1}
 
     """
@@ -320,7 +322,7 @@ def merge_all(guesses, append=None):
         result.update_highest_confidence(g)
 
     # delete very unlikely values
-    for p in result.keys():
+    for p in list(result.keys()):
         if result.confidence(p) < 0.05:
             del result[p]
 

libs/guessit/hash_ed2k.py

@@ -18,6 +18,8 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
+from __future__ import unicode_literals
+from guessit import s, to_hex
 import hashlib
 import os.path
 
@@ -25,7 +27,7 @@ import os.path
 def hash_file(filename):
     """Returns the ed2k hash of a given file.
 
-    >>> hash_file('test/dummy.srt')
+    >>> s(hash_file('tests/dummy.srt'))
     'ed2k://|file|dummy.srt|44|1CA0B9DED3473B926AA93A0A546138BB|/'
     """
     return 'ed2k://|file|%s|%d|%s|/' % (os.path.basename(filename),
@@ -58,6 +60,6 @@ def hash_filehash(filename):
         a = gen(f)
         hashes = [md4_hash(data).digest() for data in a]
         if len(hashes) == 1:
-            return hashes[0].encode("hex")
+            return to_hex(hashes[0])
         else:
             return md4_hash(reduce(lambda a, d: a + d, hashes, "")).hexd

libs/guessit/hash_mpc.py

@@ -18,6 +18,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
+from __future__ import unicode_literals
 import struct
 import os
 

libs/guessit/language.py

@@ -19,9 +19,9 @@
 #
 
 from __future__ import unicode_literals
-from guessit import fileutils
+from guessit import UnicodeMixin, base_text_type, u, s
+from guessit.fileutils import load_file_in_same_dir
 from guessit.country import Country
-from guessit.textutils import to_unicode
 import re
 import logging
 
@@ -39,8 +39,7 @@ log = logging.getLogger(__name__)
 # "An alpha-3 (bibliographic) code, an alpha-3 (terminologic) code (when given),
 # an alpha-2 code (when given), an English name, and a French name of a language
 # are all separated by pipe (|) characters."
-_iso639_contents = fileutils.load_file_in_same_dir(__file__,
-                                                   'ISO-639-2_utf-8.txt').decode('utf-8')
+_iso639_contents = load_file_in_same_dir(__file__, 'ISO-639-2_utf-8.txt')
 
 # drop the BOM from the beginning of the file
 _iso639_contents = _iso639_contents[1:]
@@ -116,7 +115,6 @@ lng_exceptions = { 'unknown': ('und', None),
                    'cn': ('chi', None),
                    'chs': ('chi', None),
                    'jp': ('jpn', None),
-                   'scc': ('srp', None),
                    'scr': ('hrv', None)
                    }
 
@@ -137,7 +135,7 @@ def lang_set(languages, strict=False):
     return set(Language(l, strict=strict) for l in languages)
 
 
-class Language(object):
+class Language(UnicodeMixin):
     """This class represents a human language.
 
     You can initialize it with pretty much anything, as it knows conversion
@@ -154,30 +152,30 @@ class Language(object):
     >>> Language('fr')
     Language(French)
 
-    >>> Language('eng').french_name
-    u'anglais'
+    >>> s(Language('eng').french_name)
+    'anglais'
 
-    >>> Language('pt(br)').country.english_name
-    u'Brazil'
+    >>> s(Language('pt(br)').country.english_name)
+    'Brazil'
 
-    >>> Language('Español (Latinoamérica)').country.english_name
-    u'Latin America'
+    >>> s(Language('Español (Latinoamérica)').country.english_name)
+    'Latin America'
 
     >>> Language('Spanish (Latin America)') == Language('Español (Latinoamérica)')
     True
 
-    >>> Language('zz', strict=False).english_name
-    u'Undetermined'
+    >>> s(Language('zz', strict=False).english_name)
+    'Undetermined'
 
-    >>> Language('pt(br)').opensubtitles
-    u'pob'
+    >>> s(Language('pt(br)').opensubtitles)
+    'pob'
     """
 
     _with_country_regexp = re.compile('(.*)\((.*)\)')
     _with_country_regexp2 = re.compile('(.*)-(.*)')
 
     def __init__(self, language, country=None, strict=False, scheme=None):
-        language = to_unicode(language.strip().lower())
+        language = u(language.strip().lower())
         with_country = (Language._with_country_regexp.match(language) or
                         Language._with_country_regexp2.match(language))
         if with_country:
@@ -249,7 +247,7 @@ class Language(object):
     def opensubtitles(self):
         if self.lang == 'por' and self.country and self.country.alpha2 == 'br':
             return 'pob'
-        elif self.lang in ['gre', 'eus', 'ice', 'srp']:
+        elif self.lang in ['gre', 'srp']:
             return self.alpha3term
         return self.alpha3
 
@@ -266,7 +264,7 @@ class Language(object):
         if isinstance(other, Language):
             return self.lang == other.lang
 
-        if isinstance(other, basestring):
+        if isinstance(other, base_text_type):
             try:
                 return self == Language(other)
             except ValueError:
@@ -286,9 +284,6 @@ class Language(object):
         else:
             return self.english_name
 
-    def __str__(self):
-        return unicode(self).encode('utf-8')
-
     def __repr__(self):
         if self.country:
             return 'Language(%s, country=%s)' % (self.english_name, self.country)

libs/guessit/matcher.py

@@ -18,8 +18,9 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
+from __future__ import unicode_literals
+from guessit import PY3, u
 from guessit.matchtree import MatchTree
-from guessit.textutils import to_utf8
 from guessit.guess import (merge_similar_guesses, merge_all,
                            choose_int, choose_string)
 import copy
@@ -69,12 +70,12 @@ class IterativeMatcher(object):
         """
 
         valid_filetypes = ('autodetect', 'subtitle', 'video',
-                            'movie', 'moviesubtitle',
-                            'episode', 'episodesubtitle')
+                           'movie', 'moviesubtitle',
+                           'episode', 'episodesubtitle')
         if filetype not in valid_filetypes:
             raise ValueError("filetype needs to be one of %s" % valid_filetypes)
-        if not isinstance(filename, unicode):
-            log.debug('WARNING: given filename to matcher is not unicode...')
+        if not PY3 and not isinstance(filename, unicode):
+            log.warning('Given filename to matcher is not unicode...')
 
         self.match_tree = MatchTree(filename)
         mtree = self.match_tree
@@ -99,21 +100,26 @@ class IterativeMatcher(object):
         apply_transfo('split_explicit_groups')
 
         # 4- try to match information for specific patterns
+        # NOTE: order needs to comply to the following:
+        #       - website before language (eg: tvu.org.ru vs russian)
+        #       - language before episodes_rexps
+        #       - properties before language (eg: he-aac vs hebrew)
+        #       - release_group before properties (eg: XviD-?? vs xvid)
         if mtree.guess['type'] in ('episode', 'episodesubtitle'):
-            strategy = ['guess_date', 'guess_video_rexps',
-                        'guess_episodes_rexps', 'guess_website',
-                        'guess_release_group', 'guess_properties',
-                        'guess_weak_episodes_rexps', 'guess_language']
+            strategy = [ 'guess_date', 'guess_website', 'guess_release_group',
+                         'guess_properties', 'guess_language',
+                         'guess_video_rexps',
+                         'guess_episodes_rexps', 'guess_weak_episodes_rexps' ]
         else:
-            strategy = ['guess_date', 'guess_video_rexps',
-                        'guess_website', 'guess_release_group',
-                        'guess_properties', 'guess_language']
+            strategy = [ 'guess_date', 'guess_website', 'guess_release_group',
+                         'guess_properties', 'guess_language',
+                         'guess_video_rexps' ]
 
         for name in strategy:
             apply_transfo(name)
 
         # more guessers for both movies and episodes
-        for name in ['guess_bonus_features']:
+        for name in ['guess_bonus_features', 'guess_year', 'guess_country']:
             apply_transfo(name)
 
         # split into '-' separated subgroups (with required separator chars
@@ -125,13 +131,12 @@ class IterativeMatcher(object):
         if mtree.guess['type'] in ('episode', 'episodesubtitle'):
             apply_transfo('guess_episode_info_from_position')
         else:
-            apply_transfo('guess_year')
             apply_transfo('guess_movie_title_from_position')
 
         # 6- perform some post-processing steps
         apply_transfo('post_process')
 
-        log.debug('Found match tree:\n%s' % (to_utf8(unicode(mtree))))
+        log.debug('Found match tree:\n%s' % u(mtree))
 
     def matched(self):
         # we need to make a copy here, as the merge functions work in place and
@@ -147,9 +152,12 @@ class IterativeMatcher(object):
 
         for string_part in ('title', 'series', 'container', 'format',
                             'releaseGroup', 'website', 'audioCodec',
-                            'videoCodec', 'screenSize', 'episodeFormat'):
+                            'videoCodec', 'screenSize', 'episodeFormat',
+                            'audioChannels'):
             merge_similar_guesses(parts, string_part, choose_string)
 
+        # 2- merge the rest, potentially discarding information not properly
+        #    merged before
         result = merge_all(parts,
                            append=['language', 'subtitleLanguage', 'other'])
 

libs/guessit/matchtree.py

@@ -18,15 +18,16 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
-from guessit import Guess
-from guessit.textutils import clean_string, str_fill, to_utf8
+from __future__ import unicode_literals
+from guessit import UnicodeMixin, base_text_type, Guess
+from guessit.textutils import clean_string, str_fill
 from guessit.patterns import group_delimiters
 import logging
 
 log = logging.getLogger(__name__)
 
 
-class BaseMatchTree(object):
+class BaseMatchTree(UnicodeMixin):
     """A MatchTree represents the hierarchical split of a string into its
     constituent semantic groups."""
 
@@ -154,6 +155,7 @@ class BaseMatchTree(object):
                      'extension': 'e',
                      'format': 'f',
                      'language': 'l',
+                     'country': 'C',
                      'videoCodec': 'v',
                      'audioCodec': 'a',
                      'website': 'w',
@@ -198,9 +200,6 @@ class BaseMatchTree(object):
     def __unicode__(self):
         return self.to_string()
 
-    def __str__(self):
-        return to_utf8(unicode(self))
-
 
 class MatchTree(BaseMatchTree):
     """The MatchTree contains a few "utility" methods which are not necessary
@@ -218,7 +217,7 @@ class MatchTree(BaseMatchTree):
         return list(self._unidentified_leaves(valid))
 
     def _leaves_containing(self, property_name):
-        if isinstance(property_name, basestring):
+        if isinstance(property_name, base_text_type):
             property_name = [ property_name ]
 
         for leaf in self._leaves():

libs/guessit/patterns.py

@@ -19,6 +19,8 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
+from __future__ import unicode_literals
+
 
 subtitle_exts = [ 'srt', 'idx', 'sub', 'ssa', 'txt' ]
 
@@ -40,7 +42,10 @@ episode_rexps = [ # ... Season 2 ...
                   (r'saison (?P<season>[0-9]+)', 1.0, (0, 0)),
 
                   # ... s02e13 ...
-                  (r'[Ss](?P<season>[0-9]{1,2}).{,3}(?P<episodeNumber>(?:[EeXx][0-9]{1,2})+)[^0-9]', 1.0, (0, -1)),
+                  (r'[Ss](?P<season>[0-9]{1,2}).{,3}(?P<episodeNumber>(?:[Ee][0-9]{1,2})+)[^0-9]', 1.0, (0, -1)),
+
+                  # ... s03-x02 ...
+                  (r'[Ss](?P<season>[0-9]{1,2}).{,3}(?P<bonusNumber>(?:[Xx][0-9]{1,2})+)[^0-9]', 1.0, (0, -1)),
 
                   # ... 2x13 ...
                   (r'[^0-9](?P<season>[0-9]{1,2})(?P<episodeNumber>(?:[xX][0-9]{1,2})+)[^0-9]', 0.8, (1, -1)),
@@ -53,18 +58,16 @@ episode_rexps = [ # ... Season 2 ...
                   (r'(?P<episodeNumber>[0-9]{1,3})v[23]' + sep, 0.6, (0, 0)),
 
                   # ... ep 23 ...
-                  ('ep' + sep + r'(?P<episodeNumber>[0-9]{1,2})[^0-9]', 0.7, (0, -1))
+                  ('ep' + sep + r'(?P<episodeNumber>[0-9]{1,2})[^0-9]', 0.7, (0, -1)),
+
+                  # ... e13 ... for a mini-series without a season number
+                  (r'e(?P<episodeNumber>[0-9]{1,2})[^0-9]', 0.6, (0, -1))
+
                   ]
 
 
 weak_episode_rexps = [ # ... 213 or 0106 ...
-                       (sep + r'(?P<episodeNumber>[0-9]{1,4})' + sep, (1, -1)),
-
-                       # ... 2x13 ...
-                       (sep + r'[^0-9](?P<season>[0-9]{1,2})\.(?P<episodeNumber>[0-9]{1,2})[^0-9]' + sep, (1, -1)),
-
-                       # ... e13 ... for a mini-series without a season number
-                       (r'e(?P<episodeNumber>[0-9]{1,4})[^0-9]', (0, -1)),
+                       (sep + r'(?P<episodeNumber>[0-9]{2,4})' + sep, (1, -1))
                        ]
 
 non_episode_title = [ 'extras', 'rip' ]
@@ -102,7 +105,7 @@ unlikely_series = ['series']
 
 properties = { 'format': [ 'DVDRip', 'HD-DVD', 'HDDVD', 'HDDVDRip', 'BluRay', 'Blu-ray', 'BDRip', 'BRRip',
                            'HDRip', 'DVD', 'DVDivX', 'HDTV', 'DVB', 'DVBRip', 'PDTV', 'WEBRip',
-                           'DVDSCR', 'Screener', 'VHS', 'VIDEO_TS' ],
+                           'DVDSCR', 'Screener', 'VHS', 'VIDEO_TS', 'WEB-DL', 'WEBDL' ],
 
                'screenSize': [ '720p', '720', '1080p', '1080' ],
 
@@ -116,7 +119,10 @@ properties = { 'format': [ 'DVDRip', 'HD-DVD', 'HDDVD', 'HDDVDRip', 'BluRay', 'B
                                  'CHD', 'ViTE', 'TLF', 'DEiTY', 'FLAiTE',
                                  'MDX', 'GM4F', 'DVL', 'SVD', 'iLUMiNADOS', ' FiNaLe',
                                  'UnSeeN', 'aXXo', 'KLAXXON', 'NoTV', 'ZeaL', 'LOL',
-                                 'HDBRiSe' ],
+                                 'SiNNERS', 'DiRTY', 'REWARD', 'ECI', 'KiNGS', 'CLUE',
+                                 'CtrlHD', 'POD', 'WiKi', 'DIMENSION', 'IMMERSE', 'FQM',
+                                 '2HD', 'REPTiLE', 'CTU', 'HALCYON', 'EbP', 'SiTV', 'SAiNTS',
+                                 'HDBRiSe', 'AlFleNi-TeaM', 'EVOLVE', '0TV' ],
 
                'episodeFormat': [ 'Minisode', 'Minisodes' ],
 
@@ -149,6 +155,7 @@ def find_properties(filename):
 property_synonyms = { 'DVD': [ 'DVDRip', 'VIDEO_TS' ],
                       'HD-DVD': [ 'HDDVD', 'HDDVDRip' ],
                       'BluRay': [ 'BDRip', 'BRRip', 'Blu-ray' ],
+                      'WEB-DL': [ 'WEBDL' ],
                       'DVB': [ 'DVBRip', 'PDTV' ],
                       'Screener': [ 'DVDSCR' ],
                       'DivX': [ 'DVDivX' ],

libs/guessit/slogging.py

@@ -18,6 +18,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
+from __future__ import unicode_literals
 import logging
 import sys
 

libs/guessit/textutils.py

@@ -18,7 +18,10 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
+from __future__ import unicode_literals
+from guessit import s
 from guessit.patterns import sep
+import functools
 import unicodedata
 import copy
 
@@ -61,45 +64,6 @@ def str_fill(string, region, c):
     return string[:start] + c * (end - start) + string[end:]
 
 
-def to_utf8(o):
-    """Convert all unicode strings found in the given object to utf-8
-    strings."""
-
-    if isinstance(o, unicode):
-        return o.encode('utf-8')
-    elif isinstance(o, list):
-        return [ to_utf8(i) for i in o ]
-    elif isinstance(o, dict):
-        # need to do it like that to handle Guess instances correctly
-        # FIXME: why is that necessary?
-        result = copy.deepcopy(o)
-        for key, value in o.items():
-            result[to_utf8(key)] = to_utf8(value)
-        return result
-
-    else:
-        return o
-
-def to_unicode(o):
-    """Convert all strings found in the given object to normalized
-    unicode strings, using the UTF-8 codec if needed."""
-
-    if isinstance(o, unicode):
-        return unicodedata.normalize('NFC', o)
-    if isinstance(o, str):
-        return unicodedata.normalize('NFC', o.decode('utf-8'))
-    elif isinstance(o, list):
-        return [ to_unicode(i) for i in o ]
-    elif isinstance(o, dict):
-        # need to do it like that to handle Guess instances correctly
-        #result = copy.deepcopy(o)
-        for key, value in o.items():
-            result[to_unicode(key)] = to_unicode(value)
-        return result
-
-    else:
-        return o
-
 
 def levenshtein(a, b):
     if not a:
@@ -176,21 +140,20 @@ def find_first_level_groups_span(string, enclosing):
 
 def split_on_groups(string, groups):
     """Split the given string using the different known groups for boundaries.
-
-    >>> split_on_groups('0123456789', [ (2, 4) ])
+    >>> s(split_on_groups('0123456789', [ (2, 4) ]))
     ['01', '23', '456789']
 
-    >>> split_on_groups('0123456789', [ (2, 4), (4, 6) ])
+    >>> s(split_on_groups('0123456789', [ (2, 4), (4, 6) ]))
     ['01', '23', '45', '6789']
 
-    >>> split_on_groups('0123456789', [ (5, 7), (2, 4) ])
+    >>> s(split_on_groups('0123456789', [ (5, 7), (2, 4) ]))
     ['01', '23', '4', '56', '789']
 
     """
     if not groups:
         return [ string ]
 
-    boundaries = sorted(set(reduce(lambda l, x: l + list(x), groups, [])))
+    boundaries = sorted(set(functools.reduce(lambda l, x: l + list(x), groups, [])))
     if boundaries[0] != 0:
         boundaries.insert(0, 0)
     if boundaries[-1] != len(string):
@@ -212,22 +175,22 @@ def find_first_level_groups(string, enclosing, blank_sep=None):
     This does not return nested groups, ie: '(ab(c)(d))' will return a single group
     containing the whole string.
 
-    >>> find_first_level_groups('', '()')
+    >>> s(find_first_level_groups('', '()'))
     ['']
 
-    >>> find_first_level_groups('abcd', '()')
+    >>> s(find_first_level_groups('abcd', '()'))
     ['abcd']
 
-    >>> find_first_level_groups('abc(de)fgh', '()')
+    >>> s(find_first_level_groups('abc(de)fgh', '()'))
     ['abc', '(de)', 'fgh']
 
-    >>> find_first_level_groups('(ab(c)(d))', '()', blank_sep = '_')
+    >>> s(find_first_level_groups('(ab(c)(d))', '()', blank_sep = '_'))
     ['_ab(c)(d)_']
 
-    >>> find_first_level_groups('ab[c]de[f]gh(i)', '[]')
+    >>> s(find_first_level_groups('ab[c]de[f]gh(i)', '[]'))
     ['ab', '[c]', 'de', '[f]', 'gh(i)']
 
-    >>> find_first_level_groups('()[]()', '()', blank_sep = '-')
+    >>> s(find_first_level_groups('()[]()', '()', blank_sep = '-'))
     ['--', '[]', '--']
 
     """

libs/guessit/transfo/__init__.py

@@ -18,7 +18,8 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
-from guessit import Guess
+from __future__ import unicode_literals
+from guessit import base_text_type, Guess
 from guessit.patterns import canonical_form
 from guessit.textutils import clean_string
 import logging
@@ -41,7 +42,7 @@ def format_guess(guess):
         if prop in ('season', 'episodeNumber', 'year', 'cdNumber',
                     'cdNumberTotal', 'bonusNumber', 'filmNumber'):
             guess[prop] = int(guess[prop])
-        elif isinstance(value, basestring):
+        elif isinstance(value, base_text_type):
             if prop in ('edition',):
                 value = clean_string(value)
             guess[prop] = canonical_form(value)
@@ -63,7 +64,7 @@ def find_and_split_node(node, strategy, logger):
 
             if isinstance(result, Guess):
                 if confidence is None:
-                    confidence = result.confidence(result.keys()[0])
+                    confidence = result.confidence(list(result.keys())[0])
             else:
                 if confidence is None:
                     confidence = 1.0

libs/guessit/transfo/guess_bonus_features.py

@@ -18,6 +18,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
+from __future__ import unicode_literals
 from guessit.transfo import found_property
 import logging
 

libs/guessit/transfo/guess_country.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# GuessIt - A library for guessing information from filenames
+# Copyright (c) 2012 Nicolas Wack <wackou@gmail.com>
+#
+# GuessIt is free software; you can redistribute it and/or modify it under
+# the terms of the Lesser GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# GuessIt is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# Lesser GNU General Public License for more details.
+#
+# You should have received a copy of the Lesser GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+from __future__ import unicode_literals
+#from guessit.transfo import SingleNodeGuesser
+#from guessit.date import search_year
+from guessit.country import Country
+from guessit import Guess
+import logging
+
+log = logging.getLogger(__name__)
+
+
+def process(mtree):
+    for node in mtree.unidentified_leaves():
+        # only keep explicit groups (enclosed in parentheses/brackets)
+        if len(node.node_idx) == 2:
+            try:
+                country = Country(node.value[1:-1], strict=True)
+                if node.value[0] + node.value[-1] not in ['()', '[]', '{}']:
+                    continue
+                node.guess = Guess(country=country, confidence=1.0)
+
+            except ValueError:
+                pass

libs/guessit/transfo/guess_date.py

@@ -18,6 +18,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
+from __future__ import unicode_literals
 from guessit.transfo import SingleNodeGuesser
 from guessit.date import search_date
 import logging

libs/guessit/transfo/guess_episode_info_from_position.py

@@ -18,6 +18,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
+from __future__ import unicode_literals
 from guessit.transfo import found_property
 from guessit.patterns import non_episode_title, unlikely_series
 import logging

libs/guessit/transfo/guess_episodes_rexps.py

@@ -18,6 +18,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
+from __future__ import unicode_literals
 from guessit import Guess
 from guessit.transfo import SingleNodeGuesser
 from guessit.patterns import episode_rexps
@@ -27,30 +28,35 @@ import logging
 log = logging.getLogger(__name__)
 
 def number_list(s):
-    return re.sub('[^0-9]+', ' ', s).split()
+    return list(re.sub('[^0-9]+', ' ', s).split())
 
 def guess_episodes_rexps(string):
     for rexp, confidence, span_adjust in episode_rexps:
         match = re.search(rexp, string, re.IGNORECASE)
         if match:
-            result = (Guess(match.groupdict(), confidence=confidence),
-                      (match.start() + span_adjust[0],
-                       match.end() + span_adjust[1]))
+            guess = Guess(match.groupdict(), confidence=confidence)
+            span = (match.start() + span_adjust[0],
+                    match.end() + span_adjust[1])
+
             # episodes which have a season > 25 are most likely errors
-            # (Simpsons is at 23!)
-            if int(result[0].get('season', 0)) > 25:
+            # (Simpsons is at 24!)
+            if int(guess.get('season', 0)) > 25:
                 continue
 
             # decide whether we have only a single episode number or an
             # episode list
-            if result[0].get('episodeNumber'):
-                eplist = number_list(result[0]['episodeNumber'])
-                result[0].set('episodeNumber', int(eplist[0]), confidence=confidence)
+            if guess.get('episodeNumber'):
+                eplist = number_list(guess['episodeNumber'])
+                guess.set('episodeNumber', int(eplist[0]), confidence=confidence)
 
                 if len(eplist) > 1:
-                    result[0].set('episodeList', map(int, eplist), confidence=confidence)
+                    guess.set('episodeList', list(map(int, eplist)), confidence=confidence)
 
-            return result
+            if guess.get('bonusNumber'):
+                eplist = number_list(guess['bonusNumber'])
+                guess.set('bonusNumber', int(eplist[0]), confidence=confidence)
+
+            return guess, span
 
     return None, None
 

libs/guessit/transfo/guess_filetype.py

@@ -18,9 +18,12 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
+from __future__ import unicode_literals
 from guessit import Guess
 from guessit.patterns import (subtitle_exts, video_exts, episode_rexps,
                               find_properties, canonical_form)
+from guessit.date import valid_year
+from guessit.textutils import clean_string
 import os.path
 import re
 import mimetypes
@@ -28,34 +31,22 @@ import logging
 
 log = logging.getLogger(__name__)
 
+# List of well known movies and series, hardcoded because they cannot be
+# guessed appropriately otherwise
+MOVIES = [ 'OSS 117' ]
+SERIES = [ 'Band of Brothers' ]
 
-def guess_filetype(filename, filetype):
-    other = {}
-
-    # look at the extension first
-    fileext = os.path.splitext(filename)[1][1:].lower()
-    if fileext in subtitle_exts:
-        if 'movie' in filetype:
-            filetype = 'moviesubtitle'
-        elif 'episode' in filetype:
-            filetype = 'episodesubtitle'
-        else:
-            filetype = 'subtitle'
-        other = { 'container': fileext }
-    elif fileext in video_exts:
-        if filetype == 'autodetect':
-            filetype = 'video'
-        other = { 'container': fileext }
-    else:
-        if filetype == 'autodetect':
-            filetype = 'unknown'
-        other = { 'extension': fileext }
+MOVIES = [ m.lower() for m in MOVIES ]
+SERIES = [ s.lower() for s in SERIES ]
 
+def guess_filetype(mtree, filetype):
     # put the filetype inside a dummy container to be able to have the
     # following functions work correctly as closures
     # this is a workaround for python 2 which doesn't have the
     # 'nonlocal' keyword (python 3 does have it)
     filetype_container = [filetype]
+    other = {}
+    filename = mtree.string
 
     def upgrade_episode():
         if filetype_container[0] == 'video':
@@ -69,14 +60,82 @@ def guess_filetype(filename, filetype):
         elif filetype_container[0] == 'subtitle':
             filetype_container[0] = 'moviesubtitle'
 
+    def upgrade_subtitle():
+        if 'movie' in filetype_container[0]:
+            filetype_container[0] = 'moviesubtitle'
+        elif 'episode' in filetype_container[0]:
+            filetype_container[0] = 'episodesubtitle'
+        else:
+            filetype_container[0] = 'subtitle'
+
+    def upgrade(type='unknown'):
+        if filetype_container[0] == 'autodetect':
+            filetype_container[0] = type
+
+
+    # look at the extension first
+    fileext = os.path.splitext(filename)[1][1:].lower()
+    if fileext in subtitle_exts:
+        upgrade_subtitle()
+        other = { 'container': fileext }
+    elif fileext in video_exts:
+        upgrade(type='video')
+        other = { 'container': fileext }
+    else:
+        upgrade(type='unknown')
+        other = { 'extension': fileext }
+
+
+
+    # check whether we are in a 'Movies', 'Tv Shows', ... folder
+    folder_rexps = [ (r'Movies?', upgrade_movie),
+                     (r'Tv ?Shows?', upgrade_episode),
+                     (r'Series', upgrade_episode)
+                     ]
+    for frexp, upgrade_func in folder_rexps:
+        frexp = re.compile(frexp, re.IGNORECASE)
+        for pathgroup in mtree.children:
+            if frexp.match(pathgroup.value):
+                upgrade_func()
+
+    # check for a few specific cases which will unintentionally make the
+    # following heuristics confused (eg: OSS 117 will look like an episode,
+    # season 1, epnum 17, when it is in fact a movie)
+    fname = clean_string(filename).lower()
+    for m in MOVIES:
+        if m in fname:
+            upgrade_movie()
+    for s in SERIES:
+        if s in fname:
+            upgrade_episode()
+
     # now look whether there are some specific hints for episode vs movie
-    if filetype in ('video', 'subtitle'):
+    if filetype_container[0] in ('video', 'subtitle'):
+        # if we have an episode_rexp (eg: s02e13), it is an episode
         for rexp, _, _ in episode_rexps:
             match = re.search(rexp, filename, re.IGNORECASE)
             if match:
                 upgrade_episode()
                 break
 
+        # if we have a 3-4 digit number that's not a year, maybe an episode
+        match = re.search(r'[^0-9]([0-9]{3,4})[^0-9]', filename)
+        if match:
+            fullnumber = int(match.group()[1:-1])
+            #season = fullnumber // 100
+            epnumber = fullnumber % 100
+            possible = True
+
+            # check for validity
+            if epnumber > 40:
+                possible = False
+            if valid_year(fullnumber):
+                possible = False
+
+            if possible:
+                upgrade_episode()
+
+        # if we have certain properties characteristic of episodes, it is an ep
         for prop, value, _, _ in find_properties(filename):
             log.debug('prop: %s = %s' % (prop, value))
             if prop == 'episodeFormat':
@@ -87,6 +146,7 @@ def guess_filetype(filename, filetype):
                 upgrade_episode()
                 break
 
+        # origin-specific type
         if 'tvu.org.ru' in filename:
             upgrade_episode()
 
@@ -98,7 +158,7 @@ def guess_filetype(filename, filetype):
 
 
 def process(mtree, filetype='autodetect'):
-    filetype, other = guess_filetype(mtree.string, filetype)
+    filetype, other = guess_filetype(mtree, filetype)
 
     mtree.guess.set('type', filetype, confidence=1.0)
     log.debug('Found with confidence %.2f: %s' % (1.0, mtree.guess))

libs/guessit/transfo/guess_language.py

@@ -18,6 +18,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
+from __future__ import unicode_literals
 from guessit import Guess
 from guessit.transfo import SingleNodeGuesser
 from guessit.language import search_language

libs/guessit/transfo/guess_movie_title_from_position.py

@@ -18,6 +18,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
+from __future__ import unicode_literals
 from guessit import Guess
 import logging
 

libs/guessit/transfo/guess_properties.py

@@ -18,6 +18,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
+from __future__ import unicode_literals
 from guessit.transfo import SingleNodeGuesser
 from guessit.patterns import find_properties
 import logging

libs/guessit/transfo/guess_release_group.py

@@ -18,24 +18,53 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
+from __future__ import unicode_literals
 from guessit.transfo import SingleNodeGuesser
+from guessit.patterns import properties, canonical_form
 import re
 import logging
 
 log = logging.getLogger(__name__)
 
 
+CODECS = properties['videoCodec']
+FORMATS = properties['format']
+
+def adjust_metadata(md):
+    codec = canonical_form(md['videoCodec'])
+    if codec in FORMATS:
+        md['format'] = codec
+        del md['videoCodec']
+    return md
+
+
 def guess_release_group(string):
     group_names = [ r'\.(Xvid)-(?P<releaseGroup>.*?)[ \.]',
                     r'\.(DivX)-(?P<releaseGroup>.*?)[\. ]',
                     r'\.(DVDivX)-(?P<releaseGroup>.*?)[\. ]',
                     ]
+
+    # first try to see whether we have both a known codec and a known release group
+    group_names = [ r'\.(?P<videoCodec>' + codec + r')-(?P<releaseGroup>.*?)[ \.]'
+                    for codec in (CODECS + FORMATS) ]
+
     for rexp in group_names:
         match = re.search(rexp, string, re.IGNORECASE)
         if match:
             metadata = match.groupdict()
-            metadata.update({ 'videoCodec': match.group(1) })
-            return metadata, (match.start() + 1, match.end() - 1)
+            if canonical_form(metadata['releaseGroup']) in properties['releaseGroup']:
+                return adjust_metadata(metadata), (match.start(1), match.end(2))
+
+    # pick anything as releaseGroup as long as we have a codec in front
+    # this doesn't include a potential dash ('-') ending the release group
+    # eg: [...].X264-HiS@SiLUHD-English.[...]
+    group_names = [ r'\.(?P<videoCodec>' + codec + r')-(?P<releaseGroup>.*?)(-(.*?))?[ \.]'
+                    for codec in (CODECS + FORMATS) ]
+
+    for rexp in group_names:
+        match = re.search(rexp, string, re.IGNORECASE)
+        if match:
+            return adjust_metadata(match.groupdict()), (match.start(1), match.end(2))
 
     return None, None
 

libs/guessit/transfo/guess_video_rexps.py

@@ -18,6 +18,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
+from __future__ import unicode_literals
 from guessit import Guess
 from guessit.transfo import SingleNodeGuesser
 from guessit.patterns import video_rexps, sep

libs/guessit/transfo/guess_weak_episodes_rexps.py

@@ -18,6 +18,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
+from __future__ import unicode_literals
 from guessit import Guess
 from guessit.transfo import SingleNodeGuesser
 from guessit.patterns import weak_episode_rexps
@@ -40,8 +41,13 @@ def guess_weak_episodes_rexps(string, node):
 
             epnum = int(metadata['episodeNumber'])
             if epnum > 100:
-                return Guess({ 'season': epnum // 100,
-                               'episodeNumber': epnum % 100 },
+                season, epnum = epnum // 100, epnum % 100
+                # episodes which have a season > 25 are most likely errors
+                # (Simpsons is at 23!)
+                if season > 25:
+                    continue
+                return Guess({ 'season': season,
+                               'episodeNumber': epnum },
                              confidence=0.6), span
             else:
                 return Guess(metadata, confidence=0.3), span

libs/guessit/transfo/guess_website.py

@@ -18,6 +18,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
+from __future__ import unicode_literals
 from guessit.transfo import SingleNodeGuesser
 from guessit.patterns import websites
 import logging

libs/guessit/transfo/guess_year.py

@@ -18,6 +18,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
+from __future__ import unicode_literals
 from guessit.transfo import SingleNodeGuesser
 from guessit.date import search_year
 import logging

libs/guessit/transfo/post_process.py

@@ -18,6 +18,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
+from __future__ import unicode_literals
 from guessit.patterns import subtitle_exts
 import logging
 

libs/guessit/transfo/split_explicit_groups.py

@@ -18,8 +18,10 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
+from __future__ import unicode_literals
 from guessit.textutils import find_first_level_groups
 from guessit.patterns import group_delimiters
+import functools
 import logging
 
 log = logging.getLogger(__name__)
@@ -33,10 +35,10 @@ def process(mtree):
         groups = find_first_level_groups(c.value, group_delimiters[0])
         for delimiters in group_delimiters:
             flatten = lambda l, x: l + find_first_level_groups(x, delimiters)
-            groups = reduce(flatten, groups, [])
+            groups = functools.reduce(flatten, groups, [])
 
         # do not do this at this moment, it is not strong enough and can break other
         # patterns, such as dates, etc...
-        #groups = reduce(lambda l, x: l + x.split('-'), groups, [])
+        #groups = functools.reduce(lambda l, x: l + x.split('-'), groups, [])
 
         c.split_on_components(groups)

libs/guessit/transfo/split_on_dash.py

@@ -18,6 +18,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
+from __future__ import unicode_literals
 from guessit.patterns import sep
 import re
 import logging

libs/guessit/transfo/split_path_components.py

@@ -18,6 +18,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
+from __future__ import unicode_literals
 from guessit import fileutils
 import os.path
 import logging

libs/requests/__init__.py

@@ -6,8 +6,35 @@
 #          /
 
 """
-requests
-~~~~~~~~
+requests HTTP library
+~~~~~~~~~~~~~~~~~~~~~
+
+Requests is an HTTP library, written in Python, for human beings. Basic GET
+usage:
+
+   >>> import requests
+   >>> r = requests.get('http://python.org')
+   >>> r.status_code
+   200
+   >>> 'Python is a programming language' in r.content
+   True
+
+... or POST:
+
+   >>> payload = dict(key1='value1', key2='value2')
+   >>> r = requests.post("http://httpbin.org/post", data=payload)
+   >>> print r.text
+   {
+     ...
+     "form": {
+       "key2": "value2",
+       "key1": "value1"
+     },
+     ...
+   }
+
+The other HTTP methods are supported - see `requests.api`. Full documentation
+is at <http://python-requests.org>.
 
 :copyright: (c) 2012 by Kenneth Reitz.
 :license: ISC, see LICENSE for more details.
@@ -15,8 +42,8 @@ requests
 """
 
 __title__ = 'requests'
-__version__ = '0.13.1'
-__build__ = 0x001301
+__version__ = '0.14.1'
+__build__ = 0x001401
 __author__ = 'Kenneth Reitz'
 __license__ = 'ISC'
 __copyright__ = 'Copyright 2012 Kenneth Reitz'

libs/requests/_oauth.py

@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+
+"""
+requests._oauth
+~~~~~~~~~~~~~~~
+
+This module contains the path hack necessary for oauthlib to be vendored into
+requests while allowing upstream changes.
+"""
+
+import os
+import sys
+
+try:
+    from oauthlib.oauth1 import rfc5849
+    from oauthlib.common import extract_params
+    from oauthlib.oauth1.rfc5849 import (Client, SIGNATURE_HMAC, SIGNATURE_TYPE_AUTH_HEADER)
+except ImportError:
+    directory = os.path.dirname(__file__)
+    path = os.path.join(directory, 'packages')
+    sys.path.insert(0, path)
+    from oauthlib.oauth1 import rfc5849
+    from oauthlib.common import extract_params
+    from oauthlib.oauth1.rfc5849 import (Client, SIGNATURE_HMAC, SIGNATURE_TYPE_AUTH_HEADER)

libs/requests/api.py

@@ -14,6 +14,7 @@ This module implements the Requests API.
 from . import sessions
 from .safe_mode import catch_exceptions_if_in_safe_mode
 
+
 @catch_exceptions_if_in_safe_mode
 def request(method, url, **kwargs):
     """Constructs and sends a :class:`Request <Request>`.
@@ -38,9 +39,19 @@ def request(method, url, **kwargs):
     :param cert: (optional) if String, path to ssl client cert file (.pem). If Tuple, ('cert', 'key') pair.
     """
 
-    s = kwargs.pop('session') if 'session' in kwargs else sessions.session()
-    return s.request(method=method, url=url, **kwargs)
+    # if this session was passed in, leave it open (and retain pooled connections);
+    # if we're making it just for this call, then close it when we're done.
+    adhoc_session = False
+    session = kwargs.pop('session', None)
+    if session is None:
+        session = sessions.session()
+        adhoc_session = True
 
+    try:
+        return session.request(method=method, url=url, **kwargs)
+    finally:
+        if adhoc_session:
+            session.close()
 
 
 def get(url, **kwargs):

libs/requests/auth.py

@@ -8,8 +8,10 @@ This module contains the authentication handlers for Requests.
 """
 
 import os
+import re
 import time
 import hashlib
+import logging
 
 from base64 import b64encode
 
@@ -17,15 +19,22 @@ from .compat import urlparse, str
 from .utils import parse_dict_header
 
 try:
-    from oauthlib.oauth1.rfc5849 import (Client, SIGNATURE_HMAC, SIGNATURE_TYPE_AUTH_HEADER)
-    from oauthlib.common import extract_params
-    # hush pyflakes:
-    SIGNATURE_HMAC; SIGNATURE_TYPE_AUTH_HEADER
+    from ._oauth import (Client, SIGNATURE_HMAC, SIGNATURE_TYPE_AUTH_HEADER, extract_params)
+
 except (ImportError, SyntaxError):
     SIGNATURE_HMAC = None
     SIGNATURE_TYPE_AUTH_HEADER = None
 
+try:
+    import kerberos as k
+except ImportError as exc:
+    k = None
+
+log = logging.getLogger(__name__)
+
 CONTENT_TYPE_FORM_URLENCODED = 'application/x-www-form-urlencoded'
+CONTENT_TYPE_MULTI_PART = 'multipart/form-data'
+
 
 def _basic_auth_str(username, password):
     """Returns a Basic Auth string."""
@@ -66,29 +75,37 @@ class OAuth1(AuthBase):
         Parameters may be included from the body if the content-type is
         urlencoded, if no content type is set an educated guess is made.
         """
-        contenttype = r.headers.get('Content-Type', None)
+        # split(";") because Content-Type may be "multipart/form-data; boundary=xxxxx"
+        contenttype = r.headers.get('Content-Type', '').split(";")[0].lower()
         # extract_params will not give params unless the body is a properly
         # formatted string, a dictionary or a list of 2-tuples.
-        decoded_body = extract_params(r.data)     
-        if contenttype == None and decoded_body != None:
-            # extract_params can only check the present r.data and does not know
-            # of r.files, thus an extra check is performed. We know that 
-            # if files are present the request will not have 
-            # Content-type: x-www-form-urlencoded. We guess it will have
-            # a mimetype of multipart/form-encoded and if this is not the case
-            # we assume the correct header will be set later.
-            if r.files:
-                # Omit body data in the signing and since it will always
-                # be empty (cant add paras to body if multipart) and we wish
-                # to preserve body. 
-                r.headers['Content-Type'] = 'multipart/form-encoded'
-                r.url, r.headers, _ = self.client.sign(
-                    unicode(r.url), unicode(r.method), None, r.headers)
-            else:
-                # Normal signing
-                r.headers['Content-Type'] = 'application/x-www-form-urlencoded'
-                r.url, r.headers, r.data = self.client.sign(
-                    unicode(r.url), unicode(r.method), r.data, r.headers)
+        decoded_body = extract_params(r.data)
+
+        # extract_params can only check the present r.data and does not know
+        # of r.files, thus an extra check is performed. We know that
+        # if files are present the request will not have
+        # Content-type: x-www-form-urlencoded. We guess it will have
+        # a mimetype of multipart/form-data and if this is not the case
+        # we assume the correct header will be set later.
+        _oauth_signed = True
+        if r.files and contenttype == CONTENT_TYPE_MULTI_PART:
+            # Omit body data in the signing and since it will always
+            # be empty (cant add paras to body if multipart) and we wish
+            # to preserve body.
+            r.url, r.headers, _ = self.client.sign(
+                unicode(r.full_url), unicode(r.method), None, r.headers)
+        elif decoded_body != None and contenttype in (CONTENT_TYPE_FORM_URLENCODED, ''):
+            # Normal signing
+            if not contenttype:
+                r.headers['Content-Type'] = CONTENT_TYPE_FORM_URLENCODED
+            r.url, r.headers, r.data = self.client.sign(
+                unicode(r.full_url), unicode(r.method), r.data, r.headers)
+        else:
+            _oauth_signed = False
+        if _oauth_signed:
+            # Both flows add params to the URL by using r.full_url,
+            # so this prevents adding it again later
+            r.params = {}
 
             # Having the authorization header, key or value, in unicode will
             # result in UnicodeDecodeErrors when the request is concatenated
@@ -100,12 +117,12 @@ class OAuth1(AuthBase):
             # >>> d
             # { u'a' : 'foo' }
             u_header = unicode('Authorization')
-            if u_header in r.headers:               
+            if u_header in r.headers:
                 auth_header = r.headers[u_header].encode('utf-8')
                 del r.headers[u_header]
                 r.headers['Authorization'] = auth_header
 
-            return r
+        return r
 
 
 class HTTPBasicAuth(AuthBase):
@@ -131,91 +148,101 @@ class HTTPDigestAuth(AuthBase):
     def __init__(self, username, password):
         self.username = username
         self.password = password
+        self.last_nonce = ''
+        self.nonce_count = 0
+        self.chal = {}
+
+    def build_digest_header(self, method, url):
+
+        realm = self.chal['realm']
+        nonce = self.chal['nonce']
+        qop = self.chal.get('qop')
+        algorithm = self.chal.get('algorithm', 'MD5')
+        opaque = self.chal.get('opaque', None)
+
+        algorithm = algorithm.upper()
+        # lambdas assume digest modules are imported at the top level
+        if algorithm == 'MD5':
+            def md5_utf8(x):
+                if isinstance(x, str):
+                    x = x.encode('utf-8')
+                return hashlib.md5(x).hexdigest()
+            hash_utf8 = md5_utf8
+        elif algorithm == 'SHA':
+            def sha_utf8(x):
+                if isinstance(x, str):
+                    x = x.encode('utf-8')
+                return hashlib.sha1(x).hexdigest()
+            hash_utf8 = sha_utf8
+        # XXX MD5-sess
+        KD = lambda s, d: hash_utf8("%s:%s" % (s, d))
+
+        if hash_utf8 is None:
+            return None
+
+        # XXX not implemented yet
+        entdig = None
+        p_parsed = urlparse(url)
+        path = p_parsed.path
+        if p_parsed.query:
+            path += '?' + p_parsed.query
+
+        A1 = '%s:%s:%s' % (self.username, realm, self.password)
+        A2 = '%s:%s' % (method, path)
+
+        if qop == 'auth':
+            if nonce == self.last_nonce:
+                self.nonce_count += 1
+            else:
+                self.nonce_count = 1
+
+            ncvalue = '%08x' % self.nonce_count
+            s = str(self.nonce_count).encode('utf-8')
+            s += nonce.encode('utf-8')
+            s += time.ctime().encode('utf-8')
+            s += os.urandom(8)
+
+            cnonce = (hashlib.sha1(s).hexdigest()[:16])
+            noncebit = "%s:%s:%s:%s:%s" % (nonce, ncvalue, cnonce, qop, hash_utf8(A2))
+            respdig = KD(hash_utf8(A1), noncebit)
+        elif qop is None:
+            respdig = KD(hash_utf8(A1), "%s:%s" % (nonce, hash_utf8(A2)))
+        else:
+            # XXX handle auth-int.
+            return None
+
+        self.last_nonce = nonce
+
+        # XXX should the partial digests be encoded too?
+        base = 'username="%s", realm="%s", nonce="%s", uri="%s", ' \
+           'response="%s"' % (self.username, realm, nonce, path, respdig)
+        if opaque:
+            base += ', opaque="%s"' % opaque
+        if entdig:
+            base += ', digest="%s"' % entdig
+            base += ', algorithm="%s"' % algorithm
+        if qop:
+            base += ', qop=auth, nc=%s, cnonce="%s"' % (ncvalue, cnonce)
+
+        return 'Digest %s' % (base)
 
     def handle_401(self, r):
         """Takes the given response and tries digest-auth, if needed."""
 
-        r.request.deregister_hook('response', self.handle_401)
+        num_401_calls = r.request.hooks['response'].count(self.handle_401)
 
         s_auth = r.headers.get('www-authenticate', '')
 
-        if 'digest' in s_auth.lower():
+        if 'digest' in s_auth.lower() and num_401_calls < 2:
 
-            last_nonce = ''
-            nonce_count = 0
+            self.chal = parse_dict_header(s_auth.replace('Digest ', ''))
 
-            chal = parse_dict_header(s_auth.replace('Digest ', ''))
+            # Consume content and release the original connection
+            # to allow our new request to reuse the same one.
+            r.content
+            r.raw.release_conn()
 
-            realm = chal['realm']
-            nonce = chal['nonce']
-            qop = chal.get('qop')
-            algorithm = chal.get('algorithm', 'MD5')
-            opaque = chal.get('opaque', None)
-
-            algorithm = algorithm.upper()
-            # lambdas assume digest modules are imported at the top level
-            if algorithm == 'MD5':
-                def md5_utf8(x):
-                    if isinstance(x, str):
-                        x = x.encode('utf-8')
-                    return hashlib.md5(x).hexdigest()
-                hash_utf8 = md5_utf8
-            elif algorithm == 'SHA':
-                def sha_utf8(x):
-                    if isinstance(x, str):
-                        x = x.encode('utf-8')
-                    return hashlib.sha1(x).hexdigest()
-                hash_utf8 = sha_utf8
-            # XXX MD5-sess
-            KD = lambda s, d: hash_utf8("%s:%s" % (s, d))
-
-            if hash_utf8 is None:
-                return None
-
-            # XXX not implemented yet
-            entdig = None
-            p_parsed = urlparse(r.request.url)
-            path = p_parsed.path
-            if p_parsed.query:
-                path += '?' + p_parsed.query
-
-            A1 = '%s:%s:%s' % (self.username, realm, self.password)
-            A2 = '%s:%s' % (r.request.method, path)
-
-            if qop == 'auth':
-                if nonce == last_nonce:
-                    nonce_count += 1
-                else:
-                    nonce_count = 1
-                    last_nonce = nonce
-
-                ncvalue = '%08x' % nonce_count
-                s = str(nonce_count).encode('utf-8')
-                s += nonce.encode('utf-8')
-                s += time.ctime().encode('utf-8')
-                s += os.urandom(8)
-
-                cnonce = (hashlib.sha1(s).hexdigest()[:16])
-                noncebit = "%s:%s:%s:%s:%s" % (nonce, ncvalue, cnonce, qop, hash_utf8(A2))
-                respdig = KD(hash_utf8(A1), noncebit)
-            elif qop is None:
-                respdig = KD(hash_utf8(A1), "%s:%s" % (nonce, hash_utf8(A2)))
-            else:
-                # XXX handle auth-int.
-                return None
-
-            # XXX should the partial digests be encoded too?
-            base = 'username="%s", realm="%s", nonce="%s", uri="%s", ' \
-                   'response="%s"' % (self.username, realm, nonce, path, respdig)
-            if opaque:
-                base += ', opaque="%s"' % opaque
-            if entdig:
-                base += ', digest="%s"' % entdig
-            base += ', algorithm="%s"' % algorithm
-            if qop:
-                base += ', qop=auth, nc=%s, cnonce="%s"' % (ncvalue, cnonce)
-
-            r.request.headers['Authorization'] = 'Digest %s' % (base)
+            r.request.headers['Authorization'] = self.build_digest_header(r.request.method, r.request.url)
             r.request.send(anyway=True)
             _r = r.request.response
             _r.history.append(r)
@@ -225,5 +252,131 @@ class HTTPDigestAuth(AuthBase):
         return r
 
     def __call__(self, r):
+        # If we have a saved nonce, skip the 401
+        if self.last_nonce:
+            r.headers['Authorization'] = self.build_digest_header(r.method, r.url)
         r.register_hook('response', self.handle_401)
         return r
+
+
+def _negotiate_value(r):
+    """Extracts the gssapi authentication token from the appropriate header"""
+
+    authreq = r.headers.get('www-authenticate', None)
+
+    if authreq:
+        rx = re.compile('(?:.*,)*\s*Negotiate\s*([^,]*),?', re.I)
+        mo = rx.search(authreq)
+        if mo:
+            return mo.group(1)
+
+    return None
+
+
+class HTTPKerberosAuth(AuthBase):
+    """Attaches HTTP GSSAPI/Kerberos Authentication to the given Request object."""
+    def __init__(self, require_mutual_auth=True):
+        if k is None:
+            raise Exception("Kerberos libraries unavailable")
+        self.context = None
+        self.require_mutual_auth = require_mutual_auth
+
+    def generate_request_header(self, r):
+        """Generates the gssapi authentication token with kerberos"""
+
+        host = urlparse(r.url).netloc
+        tail, _, head = host.rpartition(':')
+        domain = tail if tail else head
+
+        result, self.context = k.authGSSClientInit("HTTP@%s" % domain)
+
+        if result < 1:
+            raise Exception("authGSSClientInit failed")
+
+        result = k.authGSSClientStep(self.context, _negotiate_value(r))
+
+        if result < 0:
+            raise Exception("authGSSClientStep failed")
+
+        response = k.authGSSClientResponse(self.context)
+
+        return "Negotiate %s" % response
+
+    def authenticate_user(self, r):
+        """Handles user authentication with gssapi/kerberos"""
+
+        auth_header = self.generate_request_header(r)
+        log.debug("authenticate_user(): Authorization header: %s" % auth_header)
+        r.request.headers['Authorization'] = auth_header
+        r.request.send(anyway=True)
+        _r = r.request.response
+        _r.history.append(r)
+        log.debug("authenticate_user(): returning %s" % _r)
+        return _r
+
+    def handle_401(self, r):
+        """Handles 401's, attempts to use gssapi/kerberos authentication"""
+
+        log.debug("handle_401(): Handling: 401")
+        if _negotiate_value(r) is not None:
+            _r = self.authenticate_user(r)
+            log.debug("handle_401(): returning %s" % _r)
+            return _r
+        else:
+            log.debug("handle_401(): Kerberos is not supported")
+            log.debug("handle_401(): returning %s" % r)
+            return r
+
+    def handle_other(self, r):
+        """Handles all responses with the exception of 401s.
+
+        This is necessary so that we can authenticate responses if requested"""
+
+        log.debug("handle_other(): Handling: %d" % r.status_code)
+        self.deregister(r)
+        if self.require_mutual_auth:
+            if _negotiate_value(r) is not None:
+                log.debug("handle_other(): Authenticating the server")
+                _r = self.authenticate_server(r)
+                log.debug("handle_other(): returning %s" % _r)
+                return _r
+            else:
+                log.error("handle_other(): Mutual authentication failed")
+                raise Exception("Mutual authentication failed")
+        else:
+            log.debug("handle_other(): returning %s" % r)
+            return r
+
+    def authenticate_server(self, r):
+        """Uses GSSAPI to authenticate the server"""
+
+        log.debug("authenticate_server(): Authenticate header: %s" % _negotiate_value(r))
+        result = k.authGSSClientStep(self.context, _negotiate_value(r))
+        if  result < 1:
+            raise Exception("authGSSClientStep failed")
+        _r = r.request.response
+        log.debug("authenticate_server(): returning %s" % _r)
+        return _r
+
+    def handle_response(self, r):
+        """Takes the given response and tries kerberos-auth, as needed."""
+
+        if r.status_code == 401:
+            _r = self.handle_401(r)
+            log.debug("handle_response returning %s" % _r)
+            return _r
+        else:
+            _r = self.handle_other(r)
+            log.debug("handle_response returning %s" % _r)
+            return _r
+
+        log.debug("handle_response returning %s" % r)
+        return r
+
+    def deregister(self, r):
+        """Deregisters the response handler"""
+        r.request.deregister_hook('response', self.handle_response)
+
+    def __call__(self, r):
+        r.register_hook('response', self.handle_response)
+        return r

libs/requests/certs.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+ceritfi.py
+~~~~~~~~~~
+
+This module returns the installation location of cacert.pem.
+"""
+
+import os
+try:
+    import certifi
+except ImportError:
+    certifi = None
+
+
+def where():
+
+    if certifi:
+        return certifi.where()
+    else:
+        f = os.path.split(__file__)[0]
+        return os.path.join(f, 'cacert.pem')
+
+if __name__ == '__main__':
+    print(where())

libs/requests/compat.py

@@ -72,6 +72,10 @@ is_osx = ('darwin' in str(sys.platform).lower())
 is_hpux = ('hpux' in str(sys.platform).lower())   # Complete guess.
 is_solaris = ('solar==' in str(sys.platform).lower())   # Complete guess.
 
+try:
+    import simplejson as json
+except ImportError:
+    import json
 
 # ---------
 # Specifics
@@ -85,10 +89,17 @@ if is_py2:
     import cookielib
     from Cookie import Morsel
     from StringIO import StringIO
+    try:
+        import cchardet as chardet
+    except ImportError:
+        from .packages import chardet
+    from .packages.urllib3.packages.ordered_dict import OrderedDict
 
+    builtin_str = str
     bytes = str
     str = unicode
     basestring = basestring
+    numeric_types = (int, long, float)
 
 
 
@@ -98,8 +109,11 @@ elif is_py3:
     from http import cookiejar as cookielib
     from http.cookies import Morsel
     from io import StringIO
+    from .packages import chardet2 as chardet
+    from collections import OrderedDict
 
+    builtin_str = str
     str = str
     bytes = bytes
     basestring = (str,bytes)
-
+    numeric_types = (int, float)

libs/requests/cookies.py

@@ -14,6 +14,7 @@ try:
 except ImportError:
     import dummy_threading as threading
 
+
 class MockRequest(object):
     """Wraps a `requests.Request` to mimic a `urllib2.Request`.
 
@@ -39,7 +40,7 @@ class MockRequest(object):
     def get_origin_req_host(self):
         if self._r.response.history:
             r = self._r.response.history[0]
-            return urlparse(r).netloc
+            return urlparse(r.url).netloc
         else:
             return self.get_host()
 
@@ -66,6 +67,11 @@ class MockRequest(object):
     def get_new_headers(self):
         return self._new_headers
 
+    @property
+    def unverifiable(self):
+        return self.is_unverifiable()
+
+
 class MockResponse(object):
     """Wraps a `httplib.HTTPMessage` to mimic a `urllib.addinfourl`.
 
@@ -86,6 +92,7 @@ class MockResponse(object):
     def getheaders(self, name):
         self._headers.getheaders(name)
 
+
 def extract_cookies_to_jar(jar, request, response):
     """Extract the cookies from the response into a CookieJar.
 
@@ -99,12 +106,14 @@ def extract_cookies_to_jar(jar, request, response):
     res = MockResponse(response._original_response.msg)
     jar.extract_cookies(res, req)
 
+
 def get_cookie_header(jar, request):
     """Produce an appropriate Cookie header string to be sent with `request`, or None."""
     r = MockRequest(request)
     jar.add_cookie_header(r)
     return r.get_new_headers().get('Cookie')
 
+
 def remove_cookie_by_name(cookiejar, name, domain=None, path=None):
     """Unsets a cookie by name, by default over all domains and paths.
 
@@ -120,10 +129,12 @@ def remove_cookie_by_name(cookiejar, name, domain=None, path=None):
     for domain, path, name in clearables:
         cookiejar.clear(domain, path, name)
 
+
 class CookieConflictError(RuntimeError):
-    """There are two cookies that meet the criteria specified in the cookie jar. 
+    """There are two cookies that meet the criteria specified in the cookie jar.
     Use .get and .set and include domain and path args in order to be more specific."""
 
+
 class RequestsCookieJar(cookielib.CookieJar, collections.MutableMapping):
     """Compatibility class; is a cookielib.CookieJar, but exposes a dict interface.
 
@@ -181,7 +192,7 @@ class RequestsCookieJar(cookielib.CookieJar, collections.MutableMapping):
         for cookie in iter(self):
             values.append(cookie.value)
         return values
-    
+
     def items(self):
         """Dict-like items() that returns a list of name-value tuples from the jar.
         See keys() and values(). Allows client-code to call "dict(RequestsCookieJar)
@@ -215,14 +226,14 @@ class RequestsCookieJar(cookielib.CookieJar, collections.MutableMapping):
             if cookie.domain is not None and cookie.domain in domains:
                 return True
             domains.append(cookie.domain)
-        return False # there is only one domain in jar
+        return False  # there is only one domain in jar
 
     def get_dict(self, domain=None, path=None):
         """Takes as an argument an optional domain and path and returns a plain old
         Python dict of name-value pairs of cookies that meet the requirements."""
         dictionary = {}
         for cookie in iter(self):
-            if (domain == None or cookie.domain == domain) and (path == None 
+            if (domain == None or cookie.domain == domain) and (path == None
                                                 or cookie.path == path):
                 dictionary[cookie.name] = cookie.value
         return dictionary
@@ -244,7 +255,7 @@ class RequestsCookieJar(cookielib.CookieJar, collections.MutableMapping):
         remove_cookie_by_name(self, name)
 
     def _find(self, name, domain=None, path=None):
-        """Requests uses this method internally to get cookie values. Takes as args name 
+        """Requests uses this method internally to get cookie values. Takes as args name
         and optional domain and path. Returns a cookie.value. If there are conflicting cookies,
         _find arbitrarily chooses one. See _find_no_duplicates if you want an exception thrown
         if there are conflicting cookies."""
@@ -257,18 +268,18 @@ class RequestsCookieJar(cookielib.CookieJar, collections.MutableMapping):
         raise KeyError('name=%r, domain=%r, path=%r' % (name, domain, path))
 
     def _find_no_duplicates(self, name, domain=None, path=None):
-        """__get_item__ and get call _find_no_duplicates -- never used in Requests internally. 
-        Takes as args name and optional domain and path. Returns a cookie.value. 
-        Throws KeyError if cookie is not found and CookieConflictError if there are 
+        """__get_item__ and get call _find_no_duplicates -- never used in Requests internally.
+        Takes as args name and optional domain and path. Returns a cookie.value.
+        Throws KeyError if cookie is not found and CookieConflictError if there are
         multiple cookies that match name and optionally domain and path."""
         toReturn = None
         for cookie in iter(self):
             if cookie.name == name:
                 if domain is None or cookie.domain == domain:
                     if path is None or cookie.path == path:
-                        if toReturn != None: # if there are multiple cookies that meet passed in criteria
+                        if toReturn != None:  # if there are multiple cookies that meet passed in criteria
                             raise CookieConflictError('There are multiple cookies with name, %r' % (name))
-                        toReturn = cookie.value # we will eventually return this as long as no cookie conflict
+                        toReturn = cookie.value  # we will eventually return this as long as no cookie conflict
 
         if toReturn:
             return toReturn
@@ -291,6 +302,7 @@ class RequestsCookieJar(cookielib.CookieJar, collections.MutableMapping):
         """This is not implemented. Calling this will throw an exception."""
         raise NotImplementedError
 
+
 def create_cookie(name, value, **kwargs):
     """Make a cookie from underspecified parameters.
 
@@ -326,6 +338,7 @@ def create_cookie(name, value, **kwargs):
 
     return cookielib.Cookie(**result)
 
+
 def morsel_to_cookie(morsel):
     """Convert a Morsel object into a Cookie containing the one k/v pair."""
     c = create_cookie(
@@ -349,6 +362,7 @@ def morsel_to_cookie(morsel):
         )
     return c
 
+
 def cookiejar_from_dict(cookie_dict, cookiejar=None):
     """Returns a CookieJar from a key/value dictionary.
 

libs/requests/defaults.py

@@ -20,20 +20,19 @@ Configurations:
 :pool_connections: The number of active HTTP connection pools to use.
 :encode_uri: If true, URIs will automatically be percent-encoded.
 :trust_env: If true, the surrouding environment will be trusted (environ, netrc).
-:param store_cookies: If false, the received cookies as part of the HTTP response would be ignored.
+:store_cookies: If false, the received cookies as part of the HTTP response would be ignored.
 
 """
 
 SCHEMAS = ['http', 'https']
 
-from . import __version__
+from .utils import default_user_agent
 
 defaults = dict()
 
-
 defaults['base_headers'] = {
-    'User-Agent': 'python-requests/%s' % __version__,
-    'Accept-Encoding': ', '.join(('identity', 'deflate', 'compress', 'gzip')),
+    'User-Agent': default_user_agent(),
+    'Accept-Encoding': ', '.join(('gzip', 'deflate', 'compress')),
     'Accept': '*/*'
 }
 
@@ -49,5 +48,3 @@ defaults['keep_alive'] = True
 defaults['encode_uri'] = True
 defaults['trust_env'] = True
 defaults['store_cookies'] = True
-
-

libs/requests/exceptions.py

@@ -8,34 +8,44 @@ This module contains the set of Requests' exceptions.
 
 """
 
+
 class RequestException(RuntimeError):
     """There was an ambiguous exception that occurred while handling your
     request."""
 
+
 class HTTPError(RequestException):
     """An HTTP error occurred."""
     response = None
 
+
 class ConnectionError(RequestException):
     """A Connection error occurred."""
 
+
 class SSLError(ConnectionError):
     """An SSL error occurred."""
 
+
 class Timeout(RequestException):
     """The request timed out."""
 
+
 class URLRequired(RequestException):
     """A valid URL is required to make a request."""
 
+
 class TooManyRedirects(RequestException):
     """Too many redirects."""
 
+
 class MissingSchema(RequestException, ValueError):
     """The URL schema (e.g. http or https) is missing."""
 
+
 class InvalidSchema(RequestException, ValueError):
     """See defaults.py for valid schemas."""
 
+
 class InvalidURL(RequestException, ValueError):
     """ The URL provided was somehow invalid. """

libs/requests/hooks.py

@@ -25,11 +25,10 @@ Available hooks:
 
 """
 
-import traceback
-
 
 HOOKS = ('args', 'pre_request', 'pre_send', 'post_request', 'response')
 
+
 def dispatch_hook(key, hooks, hook_data):
     """Dispatches a hook dictionary on a given piece of data."""
 
@@ -42,12 +41,9 @@ def dispatch_hook(key, hooks, hook_data):
             hooks = [hooks]
 
         for hook in hooks:
-            try:
-                _hook_data = hook(hook_data)
-                if _hook_data is not None:
-                    hook_data = _hook_data
+            _hook_data = hook(hook_data)
+            if _hook_data is not None:
+                hook_data = _hook_data
 
-            except Exception:
-                traceback.print_exc()
 
     return hook_data

libs/requests/models.py

@@ -7,9 +7,10 @@ requests.models
 This module contains the primary objects that power Requests.
 """
 
-import json
 import os
+import socket
 from datetime import datetime
+from io import BytesIO
 
 from .hooks import dispatch_hook, HOOKS
 from .structures import CaseInsensitiveDict
@@ -18,6 +19,7 @@ from .status_codes import codes
 from .auth import HTTPBasicAuth, HTTPProxyAuth
 from .cookies import cookiejar_from_dict, extract_cookies_to_jar, get_cookie_header
 from .packages.urllib3.exceptions import MaxRetryError, LocationParseError
+from .packages.urllib3.exceptions import TimeoutError
 from .packages.urllib3.exceptions import SSLError as _SSLError
 from .packages.urllib3.exceptions import HTTPError as _HTTPError
 from .packages.urllib3 import connectionpool, poolmanager
@@ -29,25 +31,18 @@ from .exceptions import (
 from .utils import (
     get_encoding_from_headers, stream_untransfer, guess_filename, requote_uri,
     stream_decode_response_unicode, get_netrc_auth, get_environ_proxies,
-    DEFAULT_CA_BUNDLE_PATH)
+    to_key_val_list, DEFAULT_CA_BUNDLE_PATH, parse_header_links, iter_slices)
 from .compat import (
     cookielib, urlparse, urlunparse, urljoin, urlsplit, urlencode, str, bytes,
-    StringIO, is_py2)
-
-# Import chardet if it is available.
-try:
-    import chardet
-    # hush pyflakes
-    chardet
-except ImportError:
-    chardet = None
+    StringIO, is_py2, chardet, json, builtin_str)
 
 REDIRECT_STATI = (codes.moved, codes.found, codes.other, codes.temporary_moved)
 CONTENT_CHUNK_SIZE = 10 * 1024
 
+
 class Request(object):
-    """The :class:`Request <Request>` object. It carries out all functionality of
-    Requests. Recommended interface is with the Requests functions.
+    """The :class:`Request <Request>` object. It carries out all functionality
+    of Requests. Recommended interface is with the Requests functions.
     """
 
     def __init__(self,
@@ -65,7 +60,7 @@ class Request(object):
         proxies=None,
         hooks=None,
         config=None,
-        prefetch=False,
+        prefetch=True,
         _poolmanager=None,
         verify=None,
         session=None,
@@ -79,7 +74,14 @@ class Request(object):
         self.timeout = timeout
 
         #: Request URL.
-        self.url = url
+        #: Accept objects that have string representations.
+        try:
+            self.url = unicode(url)
+        except NameError:
+            # We're on Python 3.
+            self.url = str(url)
+        except UnicodeDecodeError:
+            self.url = url
 
         #: Dictionary of HTTP Headers to attach to the :class:`Request <Request>`.
         self.headers = dict(headers or [])
@@ -109,6 +111,10 @@ class Request(object):
         # Dictionary mapping protocol to the URL of the proxy (e.g. {'http': 'foo.bar:3128'})
         self.proxies = dict(proxies or [])
 
+        for proxy_type,uri_ref in list(self.proxies.items()):
+            if not uri_ref:
+                del self.proxies[proxy_type]
+
         # If no proxies are given, allow configuration by environment variables
         # HTTP_PROXY and HTTPS_PROXY.
         if not self.proxies and self.config.get('trust_env'):
@@ -191,7 +197,7 @@ class Request(object):
                 response.status_code = getattr(resp, 'status', None)
 
                 # Make headers case-insensitive.
-                response.headers = CaseInsensitiveDict(getattr(resp, 'headers', None))
+                response.headers = CaseInsensitiveDict(getattr(resp, 'headers', {}))
 
                 # Set encoding.
                 response.encoding = get_encoding_from_headers(response.headers)
@@ -299,7 +305,8 @@ class Request(object):
                     proxies=self.proxies,
                     verify=self.verify,
                     session=self.session,
-                    cert=self.cert
+                    cert=self.cert,
+                    prefetch=self.prefetch,
                 )
 
                 request.send()
@@ -319,53 +326,60 @@ class Request(object):
         if parameters are supplied as a dict.
         """
 
-        if isinstance(data, bytes):
-            return data
-        if isinstance(data, str):
+        if isinstance(data, (str, bytes)):
             return data
         elif hasattr(data, 'read'):
             return data
         elif hasattr(data, '__iter__'):
-            try:
-                dict(data)
-            except ValueError:
-                raise ValueError('Unable to encode lists with elements that are not 2-tuples.')
-
-            params = list(data.items() if isinstance(data, dict) else data)
             result = []
-            for k, vs in params:
+            for k, vs in to_key_val_list(data):
                 for v in isinstance(vs, list) and vs or [vs]:
-                    result.append(
-                        (k.encode('utf-8') if isinstance(k, str) else k,
-                         v.encode('utf-8') if isinstance(v, str) else v))
+                    if v is not None:
+                        result.append(
+                            (k.encode('utf-8') if isinstance(k, str) else k,
+                             v.encode('utf-8') if isinstance(v, str) else v))
             return urlencode(result, doseq=True)
         else:
             return data
 
     def _encode_files(self, files):
+        """Build the body for a multipart/form-data request.
 
+        Will successfully encode files when passed as a dict or a list of
+        2-tuples. Order is retained if data is a list of 2-tuples but abritrary
+        if parameters are supplied as a dict.
+
+        """
         if (not files) or isinstance(self.data, str):
             return None
 
-        try:
-            fields = self.data.copy()
-        except AttributeError:
-            fields = dict(self.data)
+        new_fields = []
+        fields = to_key_val_list(self.data)
+        files = to_key_val_list(files)
 
-        for (k, v) in list(files.items()):
+        for field, val in fields:
+            if isinstance(val, list):
+                for v in val:
+                    new_fields.append((field, str(v)))
+            else:
+                new_fields.append((field, str(val)))
+
+        for (k, v) in files:
             # support for explicit filename
             if isinstance(v, (tuple, list)):
                 fn, fp = v
             else:
                 fn = guess_filename(v) or k
                 fp = v
-            if isinstance(fp, (bytes, str)):
+            if isinstance(fp, str):
                 fp = StringIO(fp)
-            fields.update({k: (fn, fp.read())})
+            if isinstance(fp, bytes):
+                fp = BytesIO(fp)
+            new_fields.append((k, (fn, fp.read())))
 
-        (body, content_type) = encode_multipart_formdata(fields)
+        body, content_type = encode_multipart_formdata(new_fields)
 
-        return (body, content_type)
+        return body, content_type
 
     @property
     def full_url(self):
@@ -385,7 +399,10 @@ class Request(object):
         if not scheme in SCHEMAS:
             raise InvalidSchema("Invalid scheme %r" % scheme)
 
-        netloc = netloc.encode('idna').decode('utf-8')
+        try:
+            netloc = netloc.encode('idna').decode('utf-8')
+        except UnicodeError:
+            raise InvalidURL('URL has an invalid label.')
 
         if not path:
             path = '/'
@@ -404,14 +421,14 @@ class Request(object):
             if isinstance(fragment, str):
                 fragment = fragment.encode('utf-8')
 
-        url = (urlunparse([scheme, netloc, path, params, query, fragment]))
-
         enc_params = self._encode_params(self.params)
         if enc_params:
-            if urlparse(url).query:
-                url = '%s&%s' % (url, enc_params)
+            if query:
+                query = '%s&%s' % (query, enc_params)
             else:
-                url = '%s?%s' % (url, enc_params)
+                query = enc_params
+
+        url = (urlunparse([scheme, netloc, path, params, query, fragment]))
 
         if self.config.get('encode_uri', True):
             url = requote_uri(url)
@@ -445,8 +462,10 @@ class Request(object):
 
     def register_hook(self, event, hook):
         """Properly register a hook."""
-
-        self.hooks[event].append(hook)
+        if isinstance(hook, (list, tuple, set)):
+            self.hooks[event].extend(hook)
+        else:
+            self.hooks[event].append(hook)
 
     def deregister_hook(self, event, hook):
         """Deregister a previously registered hook.
@@ -459,7 +478,7 @@ class Request(object):
         except ValueError:
             return False
 
-    def send(self, anyway=False, prefetch=False):
+    def send(self, anyway=False, prefetch=None):
         """Sends the request. Returns True if successful, False if not.
         If there was an HTTPError during transmission,
         self.response.status_code will contain the HTTPError code.
@@ -468,6 +487,9 @@ class Request(object):
 
         :param anyway: If True, request will be sent, even if it has
         already been sent.
+
+        :param prefetch: If not None, will override the request's own setting
+        for prefetch.
         """
 
         # Build the URL
@@ -483,10 +505,6 @@ class Request(object):
                 datetime.now().isoformat(), self.method, url
             ))
 
-        # Nottin' on you.
-        body = None
-        content_type = None
-
         # Use .netrc auth if none was provided.
         if not self.auth and self.config.get('trust_env'):
             self.auth = get_netrc_auth(url)
@@ -502,6 +520,10 @@ class Request(object):
             # Update self to reflect the auth changes.
             self.__dict__.update(r.__dict__)
 
+        # Nottin' on you.
+        body = None
+        content_type = None
+
         # Multi-part file uploads.
         if self.files:
             (body, content_type) = self._encode_files(self.files)
@@ -509,7 +531,7 @@ class Request(object):
             if self.data:
 
                 body = self._encode_params(self.data)
-                if isinstance(self.data, str) or hasattr(self.data, 'read'):
+                if isinstance(self.data, str) or isinstance(self.data, builtin_str) or hasattr(self.data, 'read'):
                     content_type = None
                 else:
                     content_type = 'application/x-www-form-urlencoded'
@@ -519,10 +541,10 @@ class Request(object):
             self.headers['Content-Type'] = content_type
 
         _p = urlparse(url)
-        no_proxy = filter(lambda x:x.strip(), self.proxies.get('no', '').split(','))
+        no_proxy = filter(lambda x: x.strip(), self.proxies.get('no', '').split(','))
         proxy = self.proxies.get(_p.scheme)
 
-        if proxy and not any(map(_p.netloc.endswith, no_proxy)):
+        if proxy and not any(map(_p.hostname.endswith, no_proxy)):
             conn = poolmanager.proxy_from_url(proxy)
             _proxy = urlparse(proxy)
             if '@' in _proxy.netloc:
@@ -569,7 +591,7 @@ class Request(object):
             conn.cert_reqs = 'CERT_NONE'
             conn.ca_certs = None
 
-        if self.cert and self.verify:
+        if self.cert:
             if len(self.cert) == 2:
                 conn.cert_file = self.cert[0]
                 conn.key_file = self.cert[1]
@@ -605,14 +627,19 @@ class Request(object):
                 )
                 self.sent = True
 
+            except socket.error as sockerr:
+                raise ConnectionError(sockerr)
+
             except MaxRetryError as e:
                 raise ConnectionError(e)
 
             except (_SSLError, _HTTPError) as e:
-                if self.verify and isinstance(e, _SSLError):
+                if isinstance(e, _SSLError):
                     raise SSLError(e)
-
-                raise Timeout('Request timed out.')
+                elif isinstance(e, TimeoutError):
+                    raise Timeout(e)
+                else:
+                    raise Timeout('Request timed out.')
 
             # build_response can throw TooManyRedirects
             self._build_response(r)
@@ -625,7 +652,9 @@ class Request(object):
             self.__dict__.update(r.__dict__)
 
             # If prefetch is True, mark content as consumed.
-            if prefetch or self.prefetch:
+            if prefetch is None:
+                prefetch = self.prefetch
+            if prefetch:
                 # Save the response.
                 self.response.content
 
@@ -669,7 +698,7 @@ class Response(object):
 
         #: A list of :class:`Response <Response>` objects from
         #: the history of the Request. Any redirect responses will end
-        #: up here.
+        #: up here. The list is sorted from the oldest to the most recent request.
         self.history = []
 
         #: The :class:`Request <Request>` that created the Response.
@@ -707,9 +736,8 @@ class Response(object):
         length of each item returned as decoding can take place.
         """
         if self._content_consumed:
-            raise RuntimeError(
-                'The content for this response was already consumed'
-            )
+            # simulate reading small chunks of the content
+            return iter_slices(self._content, chunk_size)
 
         def generate():
             while 1:
@@ -773,6 +801,8 @@ class Response(object):
                 self._content = None
 
         self._content_consumed = True
+        # don't need to release the connection; that's been handled by urllib3
+        # since we exhausted the data.
         return self._content
 
     @property
@@ -798,10 +828,12 @@ class Response(object):
         # Decode unicode from given encoding.
         try:
             content = str(self.content, encoding, errors='replace')
-        except LookupError:
+        except (LookupError, TypeError):
             # A LookupError is raised if the encoding was not found which could
             # indicate a misspelling or similar mistake.
             #
+            # A TypeError can be raised if encoding is None
+            #
             # So we try blindly encoding.
             content = str(self.content, errors='replace')
 
@@ -809,29 +841,52 @@ class Response(object):
 
     @property
     def json(self):
-        """Returns the json-encoded content of a request, if any."""
+        """Returns the json-encoded content of a response, if any."""
         try:
             return json.loads(self.text or self.content)
         except ValueError:
             return None
 
+    @property
+    def links(self):
+        """Returns the parsed header links of the response, if any."""
+
+        header = self.headers['link']
+
+        # l = MultiDict()
+        l = {}
+
+        if header:
+            links = parse_header_links(header)
+
+            for link in links:
+                key = link.get('rel') or link.get('url')
+                l[key] = link
+
+        return l
+
+    @property
+    def reason(self):
+        """The HTTP Reason for the response."""
+        return self.raw.reason
+
     def raise_for_status(self, allow_redirects=True):
         """Raises stored :class:`HTTPError` or :class:`URLError`, if one occurred."""
 
         if self.error:
             raise self.error
 
-        if (self.status_code >= 300) and (self.status_code < 400) and not allow_redirects:
-            http_error = HTTPError('%s Redirection' % self.status_code)
-            http_error.response = self
-            raise http_error
+        http_error_msg = ''
+        if 300 <= self.status_code < 400 and not allow_redirects:
+            http_error_msg = '%s Redirection: %s' % (self.status_code, self.reason)
 
-        elif (self.status_code >= 400) and (self.status_code < 500):
-            http_error = HTTPError('%s Client Error' % self.status_code)
-            http_error.response = self
-            raise http_error
+        elif 400 <= self.status_code < 500:
+            http_error_msg = '%s Client Error: %s' % (self.status_code, self.reason)
 
-        elif (self.status_code >= 500) and (self.status_code < 600):
-            http_error = HTTPError('%s Server Error' % self.status_code)
+        elif 500 <= self.status_code < 600:
+            http_error_msg = '%s Server Error: %s' % (self.status_code, self.reason)
+
+        if http_error_msg:
+            http_error = HTTPError(http_error_msg)
             http_error.response = self
             raise http_error

libs/requests/packages/chardet/__init__.py

@@ -0,0 +1,26 @@
+######################## BEGIN LICENSE BLOCK ########################
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+# 
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+# 
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+__version__ = "1.0.1"
+
+def detect(aBuf):
+    import universaldetector
+    u = universaldetector.UniversalDetector()
+    u.reset()
+    u.feed(aBuf)
+    u.close()
+    return u.result

libs/requests/packages/chardet/big5freq.py

@@ -0,0 +1,923 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is Mozilla Communicator client code.
+# 
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 1998
+# the Initial Developer. All Rights Reserved.
+# 
+# Contributor(s):
+#   Mark Pilgrim - port to Python
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+# 
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+# 
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+# Big5 frequency table
+# by Taiwan's Mandarin Promotion Council 
+# <http://www.edu.tw:81/mandr/>
+# 
+# 128  --> 0.42261
+# 256  --> 0.57851
+# 512  --> 0.74851
+# 1024 --> 0.89384
+# 2048 --> 0.97583
+# 
+# Ideal Distribution Ratio = 0.74851/(1-0.74851) =2.98
+# Random Distribution Ration = 512/(5401-512)=0.105
+# 
+# Typical Distribution Ratio about 25% of Ideal one, still much higher than RDR
+
+BIG5_TYPICAL_DISTRIBUTION_RATIO = 0.75
+
+#Char to FreqOrder table
+BIG5_TABLE_SIZE = 5376
+
+Big5CharToFreqOrder = ( \
+   1,1801,1506, 255,1431, 198,   9,  82,   6,5008, 177, 202,3681,1256,2821, 110, #   16
+3814,  33,3274, 261,  76,  44,2114,  16,2946,2187,1176, 659,3971,  26,3451,2653, #   32
+1198,3972,3350,4202, 410,2215, 302, 590, 361,1964,   8, 204,  58,4510,5009,1932, #   48
+  63,5010,5011, 317,1614,  75, 222, 159,4203,2417,1480,5012,3555,3091, 224,2822, #   64
+3682,   3,  10,3973,1471,  29,2787,1135,2866,1940, 873, 130,3275,1123, 312,5013, #   80
+4511,2052, 507, 252, 682,5014, 142,1915, 124, 206,2947,  34,3556,3204,  64, 604, #   96
+5015,2501,1977,1978, 155,1991, 645, 641,1606,5016,3452, 337,  72, 406,5017,  80, #  112
+ 630, 238,3205,1509, 263, 939,1092,2654, 756,1440,1094,3453, 449,  69,2987, 591, #  128
+ 179,2096, 471, 115,2035,1844,  60,  50,2988, 134, 806,1869, 734,2036,3454, 180, #  144
+ 995,1607, 156, 537,2907, 688,5018, 319,1305, 779,2145, 514,2379, 298,4512, 359, #  160
+2502,  90,2716,1338, 663,  11, 906,1099,2553,  20,2441, 182, 532,1716,5019, 732, #  176
+1376,4204,1311,1420,3206,  25,2317,1056, 113, 399, 382,1950, 242,3455,2474, 529, #  192
+3276, 475,1447,3683,5020, 117,  21, 656, 810,1297,2300,2334,3557,5021, 126,4205, #  208
+ 706, 456, 150, 613,4513,  71,1118,2037,4206, 145,3092,  85, 835, 486,2115,1246, #  224
+1426, 428, 727,1285,1015, 800, 106, 623, 303,1281,5022,2128,2359, 347,3815, 221, #  240
+3558,3135,5023,1956,1153,4207,  83, 296,1199,3093, 192, 624,  93,5024, 822,1898, #  256
+2823,3136, 795,2065, 991,1554,1542,1592,  27,  43,2867, 859, 139,1456, 860,4514, #  272
+ 437, 712,3974, 164,2397,3137, 695, 211,3037,2097, 195,3975,1608,3559,3560,3684, #  288
+3976, 234, 811,2989,2098,3977,2233,1441,3561,1615,2380, 668,2077,1638, 305, 228, #  304
+1664,4515, 467, 415,5025, 262,2099,1593, 239, 108, 300, 200,1033, 512,1247,2078, #  320
+5026,5027,2176,3207,3685,2682, 593, 845,1062,3277,  88,1723,2038,3978,1951, 212, #  336
+ 266, 152, 149, 468,1899,4208,4516,  77, 187,5028,3038,  37,   5,2990,5029,3979, #  352
+5030,5031,  39,2524,4517,2908,3208,2079,  55, 148,  74,4518, 545, 483,1474,1029, #  368
+1665, 217,1870,1531,3138,1104,2655,4209,  24, 172,3562, 900,3980,3563,3564,4519, #  384
+  32,1408,2824,1312, 329, 487,2360,2251,2717, 784,2683,   4,3039,3351,1427,1789, #  400
+ 188, 109, 499,5032,3686,1717,1790, 888,1217,3040,4520,5033,3565,5034,3352,1520, #  416
+3687,3981, 196,1034, 775,5035,5036, 929,1816, 249, 439,  38,5037,1063,5038, 794, #  432
+3982,1435,2301,  46, 178,3278,2066,5039,2381,5040, 214,1709,4521, 804,  35, 707, #  448
+ 324,3688,1601,2554, 140, 459,4210,5041,5042,1365, 839, 272, 978,2262,2580,3456, #  464
+2129,1363,3689,1423, 697, 100,3094,  48,  70,1231, 495,3139,2196,5043,1294,5044, #  480
+2080, 462, 586,1042,3279, 853, 256, 988, 185,2382,3457,1698, 434,1084,5045,3458, #  496
+ 314,2625,2788,4522,2335,2336, 569,2285, 637,1817,2525, 757,1162,1879,1616,3459, #  512
+ 287,1577,2116, 768,4523,1671,2868,3566,2526,1321,3816, 909,2418,5046,4211, 933, #  528
+3817,4212,2053,2361,1222,4524, 765,2419,1322, 786,4525,5047,1920,1462,1677,2909, #  544
+1699,5048,4526,1424,2442,3140,3690,2600,3353,1775,1941,3460,3983,4213, 309,1369, #  560
+1130,2825, 364,2234,1653,1299,3984,3567,3985,3986,2656, 525,1085,3041, 902,2001, #  576
+1475, 964,4527, 421,1845,1415,1057,2286, 940,1364,3141, 376,4528,4529,1381,   7, #  592
+2527, 983,2383, 336,1710,2684,1846, 321,3461, 559,1131,3042,2752,1809,1132,1313, #  608
+ 265,1481,1858,5049, 352,1203,2826,3280, 167,1089, 420,2827, 776, 792,1724,3568, #  624
+4214,2443,3281,5050,4215,5051, 446, 229, 333,2753, 901,3818,1200,1557,4530,2657, #  640
+1921, 395,2754,2685,3819,4216,1836, 125, 916,3209,2626,4531,5052,5053,3820,5054, #  656
+5055,5056,4532,3142,3691,1133,2555,1757,3462,1510,2318,1409,3569,5057,2146, 438, #  672
+2601,2910,2384,3354,1068, 958,3043, 461, 311,2869,2686,4217,1916,3210,4218,1979, #  688
+ 383, 750,2755,2627,4219, 274, 539, 385,1278,1442,5058,1154,1965, 384, 561, 210, #  704
+  98,1295,2556,3570,5059,1711,2420,1482,3463,3987,2911,1257, 129,5060,3821, 642, #  720
+ 523,2789,2790,2658,5061, 141,2235,1333,  68, 176, 441, 876, 907,4220, 603,2602, #  736
+ 710, 171,3464, 404, 549,  18,3143,2398,1410,3692,1666,5062,3571,4533,2912,4534, #  752
+5063,2991, 368,5064, 146, 366,  99, 871,3693,1543, 748, 807,1586,1185,  22,2263, #  768
+ 379,3822,3211,5065,3212, 505,1942,2628,1992,1382,2319,5066, 380,2362, 218, 702, #  784
+1818,1248,3465,3044,3572,3355,3282,5067,2992,3694, 930,3283,3823,5068,  59,5069, #  800
+ 585, 601,4221, 497,3466,1112,1314,4535,1802,5070,1223,1472,2177,5071, 749,1837, #  816
+ 690,1900,3824,1773,3988,1476, 429,1043,1791,2236,2117, 917,4222, 447,1086,1629, #  832
+5072, 556,5073,5074,2021,1654, 844,1090, 105, 550, 966,1758,2828,1008,1783, 686, #  848
+1095,5075,2287, 793,1602,5076,3573,2603,4536,4223,2948,2302,4537,3825, 980,2503, #  864
+ 544, 353, 527,4538, 908,2687,2913,5077, 381,2629,1943,1348,5078,1341,1252, 560, #  880
+3095,5079,3467,2870,5080,2054, 973, 886,2081, 143,4539,5081,5082, 157,3989, 496, #  896
+4224,  57, 840, 540,2039,4540,4541,3468,2118,1445, 970,2264,1748,1966,2082,4225, #  912
+3144,1234,1776,3284,2829,3695, 773,1206,2130,1066,2040,1326,3990,1738,1725,4226, #  928
+ 279,3145,  51,1544,2604, 423,1578,2131,2067, 173,4542,1880,5083,5084,1583, 264, #  944
+ 610,3696,4543,2444, 280, 154,5085,5086,5087,1739, 338,1282,3096, 693,2871,1411, #  960
+1074,3826,2445,5088,4544,5089,5090,1240, 952,2399,5091,2914,1538,2688, 685,1483, #  976
+4227,2475,1436, 953,4228,2055,4545, 671,2400,  79,4229,2446,3285, 608, 567,2689, #  992
+3469,4230,4231,1691, 393,1261,1792,2401,5092,4546,5093,5094,5095,5096,1383,1672, # 1008
+3827,3213,1464, 522,1119, 661,1150, 216, 675,4547,3991,1432,3574, 609,4548,2690, # 1024
+2402,5097,5098,5099,4232,3045,   0,5100,2476, 315, 231,2447, 301,3356,4549,2385, # 1040
+5101, 233,4233,3697,1819,4550,4551,5102,  96,1777,1315,2083,5103, 257,5104,1810, # 1056
+3698,2718,1139,1820,4234,2022,1124,2164,2791,1778,2659,5105,3097, 363,1655,3214, # 1072
+5106,2993,5107,5108,5109,3992,1567,3993, 718, 103,3215, 849,1443, 341,3357,2949, # 1088
+1484,5110,1712, 127,  67, 339,4235,2403, 679,1412, 821,5111,5112, 834, 738, 351, # 1104
+2994,2147, 846, 235,1497,1881, 418,1993,3828,2719, 186,1100,2148,2756,3575,1545, # 1120
+1355,2950,2872,1377, 583,3994,4236,2581,2995,5113,1298,3699,1078,2557,3700,2363, # 1136
+  78,3829,3830, 267,1289,2100,2002,1594,4237, 348, 369,1274,2197,2178,1838,4552, # 1152
+1821,2830,3701,2757,2288,2003,4553,2951,2758, 144,3358, 882,4554,3995,2759,3470, # 1168
+4555,2915,5114,4238,1726, 320,5115,3996,3046, 788,2996,5116,2831,1774,1327,2873, # 1184
+3997,2832,5117,1306,4556,2004,1700,3831,3576,2364,2660, 787,2023, 506, 824,3702, # 1200
+ 534, 323,4557,1044,3359,2024,1901, 946,3471,5118,1779,1500,1678,5119,1882,4558, # 1216
+ 165, 243,4559,3703,2528, 123, 683,4239, 764,4560,  36,3998,1793, 589,2916, 816, # 1232
+ 626,1667,3047,2237,1639,1555,1622,3832,3999,5120,4000,2874,1370,1228,1933, 891, # 1248
+2084,2917, 304,4240,5121, 292,2997,2720,3577, 691,2101,4241,1115,4561, 118, 662, # 1264
+5122, 611,1156, 854,2386,1316,2875,   2, 386, 515,2918,5123,5124,3286, 868,2238, # 1280
+1486, 855,2661, 785,2216,3048,5125,1040,3216,3578,5126,3146, 448,5127,1525,5128, # 1296
+2165,4562,5129,3833,5130,4242,2833,3579,3147, 503, 818,4001,3148,1568, 814, 676, # 1312
+1444, 306,1749,5131,3834,1416,1030, 197,1428, 805,2834,1501,4563,5132,5133,5134, # 1328
+1994,5135,4564,5136,5137,2198,  13,2792,3704,2998,3149,1229,1917,5138,3835,2132, # 1344
+5139,4243,4565,2404,3580,5140,2217,1511,1727,1120,5141,5142, 646,3836,2448, 307, # 1360
+5143,5144,1595,3217,5145,5146,5147,3705,1113,1356,4002,1465,2529,2530,5148, 519, # 1376
+5149, 128,2133,  92,2289,1980,5150,4003,1512, 342,3150,2199,5151,2793,2218,1981, # 1392
+3360,4244, 290,1656,1317, 789, 827,2365,5152,3837,4566, 562, 581,4004,5153, 401, # 1408
+4567,2252,  94,4568,5154,1399,2794,5155,1463,2025,4569,3218,1944,5156, 828,1105, # 1424
+4245,1262,1394,5157,4246, 605,4570,5158,1784,2876,5159,2835, 819,2102, 578,2200, # 1440
+2952,5160,1502, 436,3287,4247,3288,2836,4005,2919,3472,3473,5161,2721,2320,5162, # 1456
+5163,2337,2068,  23,4571, 193, 826,3838,2103, 699,1630,4248,3098, 390,1794,1064, # 1472
+3581,5164,1579,3099,3100,1400,5165,4249,1839,1640,2877,5166,4572,4573, 137,4250, # 1488
+ 598,3101,1967, 780, 104, 974,2953,5167, 278, 899, 253, 402, 572, 504, 493,1339, # 1504
+5168,4006,1275,4574,2582,2558,5169,3706,3049,3102,2253, 565,1334,2722, 863,  41, # 1520
+5170,5171,4575,5172,1657,2338,  19, 463,2760,4251, 606,5173,2999,3289,1087,2085, # 1536
+1323,2662,3000,5174,1631,1623,1750,4252,2691,5175,2878, 791,2723,2663,2339, 232, # 1552
+2421,5176,3001,1498,5177,2664,2630, 755,1366,3707,3290,3151,2026,1609, 119,1918, # 1568
+3474, 862,1026,4253,5178,4007,3839,4576,4008,4577,2265,1952,2477,5179,1125, 817, # 1584
+4254,4255,4009,1513,1766,2041,1487,4256,3050,3291,2837,3840,3152,5180,5181,1507, # 1600
+5182,2692, 733,  40,1632,1106,2879, 345,4257, 841,2531, 230,4578,3002,1847,3292, # 1616
+3475,5183,1263, 986,3476,5184, 735, 879, 254,1137, 857, 622,1300,1180,1388,1562, # 1632
+4010,4011,2954, 967,2761,2665,1349, 592,2134,1692,3361,3003,1995,4258,1679,4012, # 1648
+1902,2188,5185, 739,3708,2724,1296,1290,5186,4259,2201,2202,1922,1563,2605,2559, # 1664
+1871,2762,3004,5187, 435,5188, 343,1108, 596,  17,1751,4579,2239,3477,3709,5189, # 1680
+4580, 294,3582,2955,1693, 477, 979, 281,2042,3583, 643,2043,3710,2631,2795,2266, # 1696
+1031,2340,2135,2303,3584,4581, 367,1249,2560,5190,3585,5191,4582,1283,3362,2005, # 1712
+ 240,1762,3363,4583,4584, 836,1069,3153, 474,5192,2149,2532, 268,3586,5193,3219, # 1728
+1521,1284,5194,1658,1546,4260,5195,3587,3588,5196,4261,3364,2693,1685,4262, 961, # 1744
+1673,2632, 190,2006,2203,3841,4585,4586,5197, 570,2504,3711,1490,5198,4587,2633, # 1760
+3293,1957,4588, 584,1514, 396,1045,1945,5199,4589,1968,2449,5200,5201,4590,4013, # 1776
+ 619,5202,3154,3294, 215,2007,2796,2561,3220,4591,3221,4592, 763,4263,3842,4593, # 1792
+5203,5204,1958,1767,2956,3365,3712,1174, 452,1477,4594,3366,3155,5205,2838,1253, # 1808
+2387,2189,1091,2290,4264, 492,5206, 638,1169,1825,2136,1752,4014, 648, 926,1021, # 1824
+1324,4595, 520,4596, 997, 847,1007, 892,4597,3843,2267,1872,3713,2405,1785,4598, # 1840
+1953,2957,3103,3222,1728,4265,2044,3714,4599,2008,1701,3156,1551,  30,2268,4266, # 1856
+5207,2027,4600,3589,5208, 501,5209,4267, 594,3478,2166,1822,3590,3479,3591,3223, # 1872
+ 829,2839,4268,5210,1680,3157,1225,4269,5211,3295,4601,4270,3158,2341,5212,4602, # 1888
+4271,5213,4015,4016,5214,1848,2388,2606,3367,5215,4603, 374,4017, 652,4272,4273, # 1904
+ 375,1140, 798,5216,5217,5218,2366,4604,2269, 546,1659, 138,3051,2450,4605,5219, # 1920
+2254, 612,1849, 910, 796,3844,1740,1371, 825,3845,3846,5220,2920,2562,5221, 692, # 1936
+ 444,3052,2634, 801,4606,4274,5222,1491, 244,1053,3053,4275,4276, 340,5223,4018, # 1952
+1041,3005, 293,1168,  87,1357,5224,1539, 959,5225,2240, 721, 694,4277,3847, 219, # 1968
+1478, 644,1417,3368,2666,1413,1401,1335,1389,4019,5226,5227,3006,2367,3159,1826, # 1984
+ 730,1515, 184,2840,  66,4607,5228,1660,2958, 246,3369, 378,1457, 226,3480, 975, # 2000
+4020,2959,1264,3592, 674, 696,5229, 163,5230,1141,2422,2167, 713,3593,3370,4608, # 2016
+4021,5231,5232,1186,  15,5233,1079,1070,5234,1522,3224,3594, 276,1050,2725, 758, # 2032
+1126, 653,2960,3296,5235,2342, 889,3595,4022,3104,3007, 903,1250,4609,4023,3481, # 2048
+3596,1342,1681,1718, 766,3297, 286,  89,2961,3715,5236,1713,5237,2607,3371,3008, # 2064
+5238,2962,2219,3225,2880,5239,4610,2505,2533, 181, 387,1075,4024, 731,2190,3372, # 2080
+5240,3298, 310, 313,3482,2304, 770,4278,  54,3054, 189,4611,3105,3848,4025,5241, # 2096
+1230,1617,1850, 355,3597,4279,4612,3373, 111,4280,3716,1350,3160,3483,3055,4281, # 2112
+2150,3299,3598,5242,2797,4026,4027,3009, 722,2009,5243,1071, 247,1207,2343,2478, # 2128
+1378,4613,2010, 864,1437,1214,4614, 373,3849,1142,2220, 667,4615, 442,2763,2563, # 2144
+3850,4028,1969,4282,3300,1840, 837, 170,1107, 934,1336,1883,5244,5245,2119,4283, # 2160
+2841, 743,1569,5246,4616,4284, 582,2389,1418,3484,5247,1803,5248, 357,1395,1729, # 2176
+3717,3301,2423,1564,2241,5249,3106,3851,1633,4617,1114,2086,4285,1532,5250, 482, # 2192
+2451,4618,5251,5252,1492, 833,1466,5253,2726,3599,1641,2842,5254,1526,1272,3718, # 2208
+4286,1686,1795, 416,2564,1903,1954,1804,5255,3852,2798,3853,1159,2321,5256,2881, # 2224
+4619,1610,1584,3056,2424,2764, 443,3302,1163,3161,5257,5258,4029,5259,4287,2506, # 2240
+3057,4620,4030,3162,2104,1647,3600,2011,1873,4288,5260,4289, 431,3485,5261, 250, # 2256
+  97,  81,4290,5262,1648,1851,1558, 160, 848,5263, 866, 740,1694,5264,2204,2843, # 2272
+3226,4291,4621,3719,1687, 950,2479, 426, 469,3227,3720,3721,4031,5265,5266,1188, # 2288
+ 424,1996, 861,3601,4292,3854,2205,2694, 168,1235,3602,4293,5267,2087,1674,4622, # 2304
+3374,3303, 220,2565,1009,5268,3855, 670,3010, 332,1208, 717,5269,5270,3603,2452, # 2320
+4032,3375,5271, 513,5272,1209,2882,3376,3163,4623,1080,5273,5274,5275,5276,2534, # 2336
+3722,3604, 815,1587,4033,4034,5277,3605,3486,3856,1254,4624,1328,3058,1390,4035, # 2352
+1741,4036,3857,4037,5278, 236,3858,2453,3304,5279,5280,3723,3859,1273,3860,4625, # 2368
+5281, 308,5282,4626, 245,4627,1852,2480,1307,2583, 430, 715,2137,2454,5283, 270, # 2384
+ 199,2883,4038,5284,3606,2727,1753, 761,1754, 725,1661,1841,4628,3487,3724,5285, # 2400
+5286, 587,  14,3305, 227,2608, 326, 480,2270, 943,2765,3607, 291, 650,1884,5287, # 2416
+1702,1226, 102,1547,  62,3488, 904,4629,3489,1164,4294,5288,5289,1224,1548,2766, # 2432
+ 391, 498,1493,5290,1386,1419,5291,2056,1177,4630, 813, 880,1081,2368, 566,1145, # 2448
+4631,2291,1001,1035,2566,2609,2242, 394,1286,5292,5293,2069,5294,  86,1494,1730, # 2464
+4039, 491,1588, 745, 897,2963, 843,3377,4040,2767,2884,3306,1768, 998,2221,2070, # 2480
+ 397,1827,1195,1970,3725,3011,3378, 284,5295,3861,2507,2138,2120,1904,5296,4041, # 2496
+2151,4042,4295,1036,3490,1905, 114,2567,4296, 209,1527,5297,5298,2964,2844,2635, # 2512
+2390,2728,3164, 812,2568,5299,3307,5300,1559, 737,1885,3726,1210, 885,  28,2695, # 2528
+3608,3862,5301,4297,1004,1780,4632,5302, 346,1982,2222,2696,4633,3863,1742, 797, # 2544
+1642,4043,1934,1072,1384,2152, 896,4044,3308,3727,3228,2885,3609,5303,2569,1959, # 2560
+4634,2455,1786,5304,5305,5306,4045,4298,1005,1308,3728,4299,2729,4635,4636,1528, # 2576
+2610, 161,1178,4300,1983, 987,4637,1101,4301, 631,4046,1157,3229,2425,1343,1241, # 2592
+1016,2243,2570, 372, 877,2344,2508,1160, 555,1935, 911,4047,5307, 466,1170, 169, # 2608
+1051,2921,2697,3729,2481,3012,1182,2012,2571,1251,2636,5308, 992,2345,3491,1540, # 2624
+2730,1201,2071,2406,1997,2482,5309,4638, 528,1923,2191,1503,1874,1570,2369,3379, # 2640
+3309,5310, 557,1073,5311,1828,3492,2088,2271,3165,3059,3107, 767,3108,2799,4639, # 2656
+1006,4302,4640,2346,1267,2179,3730,3230, 778,4048,3231,2731,1597,2667,5312,4641, # 2672
+5313,3493,5314,5315,5316,3310,2698,1433,3311, 131,  95,1504,4049, 723,4303,3166, # 2688
+1842,3610,2768,2192,4050,2028,2105,3731,5317,3013,4051,1218,5318,3380,3232,4052, # 2704
+4304,2584, 248,1634,3864, 912,5319,2845,3732,3060,3865, 654,  53,5320,3014,5321, # 2720
+1688,4642, 777,3494,1032,4053,1425,5322, 191, 820,2121,2846, 971,4643, 931,3233, # 2736
+ 135, 664, 783,3866,1998, 772,2922,1936,4054,3867,4644,2923,3234, 282,2732, 640, # 2752
+1372,3495,1127, 922, 325,3381,5323,5324, 711,2045,5325,5326,4055,2223,2800,1937, # 2768
+4056,3382,2224,2255,3868,2305,5327,4645,3869,1258,3312,4057,3235,2139,2965,4058, # 2784
+4059,5328,2225, 258,3236,4646, 101,1227,5329,3313,1755,5330,1391,3314,5331,2924, # 2800
+2057, 893,5332,5333,5334,1402,4305,2347,5335,5336,3237,3611,5337,5338, 878,1325, # 2816
+1781,2801,4647, 259,1385,2585, 744,1183,2272,4648,5339,4060,2509,5340, 684,1024, # 2832
+4306,5341, 472,3612,3496,1165,3315,4061,4062, 322,2153, 881, 455,1695,1152,1340, # 2848
+ 660, 554,2154,4649,1058,4650,4307, 830,1065,3383,4063,4651,1924,5342,1703,1919, # 2864
+5343, 932,2273, 122,5344,4652, 947, 677,5345,3870,2637, 297,1906,1925,2274,4653, # 2880
+2322,3316,5346,5347,4308,5348,4309,  84,4310, 112, 989,5349, 547,1059,4064, 701, # 2896
+3613,1019,5350,4311,5351,3497, 942, 639, 457,2306,2456, 993,2966, 407, 851, 494, # 2912
+4654,3384, 927,5352,1237,5353,2426,3385, 573,4312, 680, 921,2925,1279,1875, 285, # 2928
+ 790,1448,1984, 719,2168,5354,5355,4655,4065,4066,1649,5356,1541, 563,5357,1077, # 2944
+5358,3386,3061,3498, 511,3015,4067,4068,3733,4069,1268,2572,3387,3238,4656,4657, # 2960
+5359, 535,1048,1276,1189,2926,2029,3167,1438,1373,2847,2967,1134,2013,5360,4313, # 2976
+1238,2586,3109,1259,5361, 700,5362,2968,3168,3734,4314,5363,4315,1146,1876,1907, # 2992
+4658,2611,4070, 781,2427, 132,1589, 203, 147, 273,2802,2407, 898,1787,2155,4071, # 3008
+4072,5364,3871,2803,5365,5366,4659,4660,5367,3239,5368,1635,3872, 965,5369,1805, # 3024
+2699,1516,3614,1121,1082,1329,3317,4073,1449,3873,  65,1128,2848,2927,2769,1590, # 3040
+3874,5370,5371,  12,2668,  45, 976,2587,3169,4661, 517,2535,1013,1037,3240,5372, # 3056
+3875,2849,5373,3876,5374,3499,5375,2612, 614,1999,2323,3877,3110,2733,2638,5376, # 3072
+2588,4316, 599,1269,5377,1811,3735,5378,2700,3111, 759,1060, 489,1806,3388,3318, # 3088
+1358,5379,5380,2391,1387,1215,2639,2256, 490,5381,5382,4317,1759,2392,2348,5383, # 3104
+4662,3878,1908,4074,2640,1807,3241,4663,3500,3319,2770,2349, 874,5384,5385,3501, # 3120
+3736,1859,  91,2928,3737,3062,3879,4664,5386,3170,4075,2669,5387,3502,1202,1403, # 3136
+3880,2969,2536,1517,2510,4665,3503,2511,5388,4666,5389,2701,1886,1495,1731,4076, # 3152
+2370,4667,5390,2030,5391,5392,4077,2702,1216, 237,2589,4318,2324,4078,3881,4668, # 3168
+4669,2703,3615,3504, 445,4670,5393,5394,5395,5396,2771,  61,4079,3738,1823,4080, # 3184
+5397, 687,2046, 935, 925, 405,2670, 703,1096,1860,2734,4671,4081,1877,1367,2704, # 3200
+3389, 918,2106,1782,2483, 334,3320,1611,1093,4672, 564,3171,3505,3739,3390, 945, # 3216
+2641,2058,4673,5398,1926, 872,4319,5399,3506,2705,3112, 349,4320,3740,4082,4674, # 3232
+3882,4321,3741,2156,4083,4675,4676,4322,4677,2408,2047, 782,4084, 400, 251,4323, # 3248
+1624,5400,5401, 277,3742, 299,1265, 476,1191,3883,2122,4324,4325,1109, 205,5402, # 3264
+2590,1000,2157,3616,1861,5403,5404,5405,4678,5406,4679,2573, 107,2484,2158,4085, # 3280
+3507,3172,5407,1533, 541,1301, 158, 753,4326,2886,3617,5408,1696, 370,1088,4327, # 3296
+4680,3618, 579, 327, 440, 162,2244, 269,1938,1374,3508, 968,3063,  56,1396,3113, # 3312
+2107,3321,3391,5409,1927,2159,4681,3016,5410,3619,5411,5412,3743,4682,2485,5413, # 3328
+2804,5414,1650,4683,5415,2613,5416,5417,4086,2671,3392,1149,3393,4087,3884,4088, # 3344
+5418,1076,  49,5419, 951,3242,3322,3323, 450,2850, 920,5420,1812,2805,2371,4328, # 3360
+1909,1138,2372,3885,3509,5421,3243,4684,1910,1147,1518,2428,4685,3886,5422,4686, # 3376
+2393,2614, 260,1796,3244,5423,5424,3887,3324, 708,5425,3620,1704,5426,3621,1351, # 3392
+1618,3394,3017,1887, 944,4329,3395,4330,3064,3396,4331,5427,3744, 422, 413,1714, # 3408
+3325, 500,2059,2350,4332,2486,5428,1344,1911, 954,5429,1668,5430,5431,4089,2409, # 3424
+4333,3622,3888,4334,5432,2307,1318,2512,3114, 133,3115,2887,4687, 629,  31,2851, # 3440
+2706,3889,4688, 850, 949,4689,4090,2970,1732,2089,4335,1496,1853,5433,4091, 620, # 3456
+3245, 981,1242,3745,3397,1619,3746,1643,3326,2140,2457,1971,1719,3510,2169,5434, # 3472
+3246,5435,5436,3398,1829,5437,1277,4690,1565,2048,5438,1636,3623,3116,5439, 869, # 3488
+2852, 655,3890,3891,3117,4092,3018,3892,1310,3624,4691,5440,5441,5442,1733, 558, # 3504
+4692,3747, 335,1549,3065,1756,4336,3748,1946,3511,1830,1291,1192, 470,2735,2108, # 3520
+2806, 913,1054,4093,5443,1027,5444,3066,4094,4693, 982,2672,3399,3173,3512,3247, # 3536
+3248,1947,2807,5445, 571,4694,5446,1831,5447,3625,2591,1523,2429,5448,2090, 984, # 3552
+4695,3749,1960,5449,3750, 852, 923,2808,3513,3751, 969,1519, 999,2049,2325,1705, # 3568
+5450,3118, 615,1662, 151, 597,4095,2410,2326,1049, 275,4696,3752,4337, 568,3753, # 3584
+3626,2487,4338,3754,5451,2430,2275, 409,3249,5452,1566,2888,3514,1002, 769,2853, # 3600
+ 194,2091,3174,3755,2226,3327,4339, 628,1505,5453,5454,1763,2180,3019,4096, 521, # 3616
+1161,2592,1788,2206,2411,4697,4097,1625,4340,4341, 412,  42,3119, 464,5455,2642, # 3632
+4698,3400,1760,1571,2889,3515,2537,1219,2207,3893,2643,2141,2373,4699,4700,3328, # 3648
+1651,3401,3627,5456,5457,3628,2488,3516,5458,3756,5459,5460,2276,2092, 460,5461, # 3664
+4701,5462,3020, 962, 588,3629, 289,3250,2644,1116,  52,5463,3067,1797,5464,5465, # 3680
+5466,1467,5467,1598,1143,3757,4342,1985,1734,1067,4702,1280,3402, 465,4703,1572, # 3696
+ 510,5468,1928,2245,1813,1644,3630,5469,4704,3758,5470,5471,2673,1573,1534,5472, # 3712
+5473, 536,1808,1761,3517,3894,3175,2645,5474,5475,5476,4705,3518,2929,1912,2809, # 3728
+5477,3329,1122, 377,3251,5478, 360,5479,5480,4343,1529, 551,5481,2060,3759,1769, # 3744
+2431,5482,2930,4344,3330,3120,2327,2109,2031,4706,1404, 136,1468,1479, 672,1171, # 3760
+3252,2308, 271,3176,5483,2772,5484,2050, 678,2736, 865,1948,4707,5485,2014,4098, # 3776
+2971,5486,2737,2227,1397,3068,3760,4708,4709,1735,2931,3403,3631,5487,3895, 509, # 3792
+2854,2458,2890,3896,5488,5489,3177,3178,4710,4345,2538,4711,2309,1166,1010, 552, # 3808
+ 681,1888,5490,5491,2972,2973,4099,1287,1596,1862,3179, 358, 453, 736, 175, 478, # 3824
+1117, 905,1167,1097,5492,1854,1530,5493,1706,5494,2181,3519,2292,3761,3520,3632, # 3840
+4346,2093,4347,5495,3404,1193,2489,4348,1458,2193,2208,1863,1889,1421,3331,2932, # 3856
+3069,2182,3521, 595,2123,5496,4100,5497,5498,4349,1707,2646, 223,3762,1359, 751, # 3872
+3121, 183,3522,5499,2810,3021, 419,2374, 633, 704,3897,2394, 241,5500,5501,5502, # 3888
+ 838,3022,3763,2277,2773,2459,3898,1939,2051,4101,1309,3122,2246,1181,5503,1136, # 3904
+2209,3899,2375,1446,4350,2310,4712,5504,5505,4351,1055,2615, 484,3764,5506,4102, # 3920
+ 625,4352,2278,3405,1499,4353,4103,5507,4104,4354,3253,2279,2280,3523,5508,5509, # 3936
+2774, 808,2616,3765,3406,4105,4355,3123,2539, 526,3407,3900,4356, 955,5510,1620, # 3952
+4357,2647,2432,5511,1429,3766,1669,1832, 994, 928,5512,3633,1260,5513,5514,5515, # 3968
+1949,2293, 741,2933,1626,4358,2738,2460, 867,1184, 362,3408,1392,5516,5517,4106, # 3984
+4359,1770,1736,3254,2934,4713,4714,1929,2707,1459,1158,5518,3070,3409,2891,1292, # 4000
+1930,2513,2855,3767,1986,1187,2072,2015,2617,4360,5519,2574,2514,2170,3768,2490, # 4016
+3332,5520,3769,4715,5521,5522, 666,1003,3023,1022,3634,4361,5523,4716,1814,2257, # 4032
+ 574,3901,1603, 295,1535, 705,3902,4362, 283, 858, 417,5524,5525,3255,4717,4718, # 4048
+3071,1220,1890,1046,2281,2461,4107,1393,1599, 689,2575, 388,4363,5526,2491, 802, # 4064
+5527,2811,3903,2061,1405,2258,5528,4719,3904,2110,1052,1345,3256,1585,5529, 809, # 4080
+5530,5531,5532, 575,2739,3524, 956,1552,1469,1144,2328,5533,2329,1560,2462,3635, # 4096
+3257,4108, 616,2210,4364,3180,2183,2294,5534,1833,5535,3525,4720,5536,1319,3770, # 4112
+3771,1211,3636,1023,3258,1293,2812,5537,5538,5539,3905, 607,2311,3906, 762,2892, # 4128
+1439,4365,1360,4721,1485,3072,5540,4722,1038,4366,1450,2062,2648,4367,1379,4723, # 4144
+2593,5541,5542,4368,1352,1414,2330,2935,1172,5543,5544,3907,3908,4724,1798,1451, # 4160
+5545,5546,5547,5548,2936,4109,4110,2492,2351, 411,4111,4112,3637,3333,3124,4725, # 4176
+1561,2674,1452,4113,1375,5549,5550,  47,2974, 316,5551,1406,1591,2937,3181,5552, # 4192
+1025,2142,3125,3182, 354,2740, 884,2228,4369,2412, 508,3772, 726,3638, 996,2433, # 4208
+3639, 729,5553, 392,2194,1453,4114,4726,3773,5554,5555,2463,3640,2618,1675,2813, # 4224
+ 919,2352,2975,2353,1270,4727,4115,  73,5556,5557, 647,5558,3259,2856,2259,1550, # 4240
+1346,3024,5559,1332, 883,3526,5560,5561,5562,5563,3334,2775,5564,1212, 831,1347, # 4256
+4370,4728,2331,3909,1864,3073, 720,3910,4729,4730,3911,5565,4371,5566,5567,4731, # 4272
+5568,5569,1799,4732,3774,2619,4733,3641,1645,2376,4734,5570,2938, 669,2211,2675, # 4288
+2434,5571,2893,5572,5573,1028,3260,5574,4372,2413,5575,2260,1353,5576,5577,4735, # 4304
+3183, 518,5578,4116,5579,4373,1961,5580,2143,4374,5581,5582,3025,2354,2355,3912, # 4320
+ 516,1834,1454,4117,2708,4375,4736,2229,2620,1972,1129,3642,5583,2776,5584,2976, # 4336
+1422, 577,1470,3026,1524,3410,5585,5586, 432,4376,3074,3527,5587,2594,1455,2515, # 4352
+2230,1973,1175,5588,1020,2741,4118,3528,4737,5589,2742,5590,1743,1361,3075,3529, # 4368
+2649,4119,4377,4738,2295, 895, 924,4378,2171, 331,2247,3076, 166,1627,3077,1098, # 4384
+5591,1232,2894,2231,3411,4739, 657, 403,1196,2377, 542,3775,3412,1600,4379,3530, # 4400
+5592,4740,2777,3261, 576, 530,1362,4741,4742,2540,2676,3776,4120,5593, 842,3913, # 4416
+5594,2814,2032,1014,4121, 213,2709,3413, 665, 621,4380,5595,3777,2939,2435,5596, # 4432
+2436,3335,3643,3414,4743,4381,2541,4382,4744,3644,1682,4383,3531,1380,5597, 724, # 4448
+2282, 600,1670,5598,1337,1233,4745,3126,2248,5599,1621,4746,5600, 651,4384,5601, # 4464
+1612,4385,2621,5602,2857,5603,2743,2312,3078,5604, 716,2464,3079, 174,1255,2710, # 4480
+4122,3645, 548,1320,1398, 728,4123,1574,5605,1891,1197,3080,4124,5606,3081,3082, # 4496
+3778,3646,3779, 747,5607, 635,4386,4747,5608,5609,5610,4387,5611,5612,4748,5613, # 4512
+3415,4749,2437, 451,5614,3780,2542,2073,4388,2744,4389,4125,5615,1764,4750,5616, # 4528
+4390, 350,4751,2283,2395,2493,5617,4391,4126,2249,1434,4127, 488,4752, 458,4392, # 4544
+4128,3781, 771,1330,2396,3914,2576,3184,2160,2414,1553,2677,3185,4393,5618,2494, # 4560
+2895,2622,1720,2711,4394,3416,4753,5619,2543,4395,5620,3262,4396,2778,5621,2016, # 4576
+2745,5622,1155,1017,3782,3915,5623,3336,2313, 201,1865,4397,1430,5624,4129,5625, # 4592
+5626,5627,5628,5629,4398,1604,5630, 414,1866, 371,2595,4754,4755,3532,2017,3127, # 4608
+4756,1708, 960,4399, 887, 389,2172,1536,1663,1721,5631,2232,4130,2356,2940,1580, # 4624
+5632,5633,1744,4757,2544,4758,4759,5634,4760,5635,2074,5636,4761,3647,3417,2896, # 4640
+4400,5637,4401,2650,3418,2815, 673,2712,2465, 709,3533,4131,3648,4402,5638,1148, # 4656
+ 502, 634,5639,5640,1204,4762,3649,1575,4763,2623,3783,5641,3784,3128, 948,3263, # 4672
+ 121,1745,3916,1110,5642,4403,3083,2516,3027,4132,3785,1151,1771,3917,1488,4133, # 4688
+1987,5643,2438,3534,5644,5645,2094,5646,4404,3918,1213,1407,2816, 531,2746,2545, # 4704
+3264,1011,1537,4764,2779,4405,3129,1061,5647,3786,3787,1867,2897,5648,2018, 120, # 4720
+4406,4407,2063,3650,3265,2314,3919,2678,3419,1955,4765,4134,5649,3535,1047,2713, # 4736
+1266,5650,1368,4766,2858, 649,3420,3920,2546,2747,1102,2859,2679,5651,5652,2000, # 4752
+5653,1111,3651,2977,5654,2495,3921,3652,2817,1855,3421,3788,5655,5656,3422,2415, # 4768
+2898,3337,3266,3653,5657,2577,5658,3654,2818,4135,1460, 856,5659,3655,5660,2899, # 4784
+2978,5661,2900,3922,5662,4408, 632,2517, 875,3923,1697,3924,2296,5663,5664,4767, # 4800
+3028,1239, 580,4768,4409,5665, 914, 936,2075,1190,4136,1039,2124,5666,5667,5668, # 4816
+5669,3423,1473,5670,1354,4410,3925,4769,2173,3084,4137, 915,3338,4411,4412,3339, # 4832
+1605,1835,5671,2748, 398,3656,4413,3926,4138, 328,1913,2860,4139,3927,1331,4414, # 4848
+3029, 937,4415,5672,3657,4140,4141,3424,2161,4770,3425, 524, 742, 538,3085,1012, # 4864
+5673,5674,3928,2466,5675, 658,1103, 225,3929,5676,5677,4771,5678,4772,5679,3267, # 4880
+1243,5680,4142, 963,2250,4773,5681,2714,3658,3186,5682,5683,2596,2332,5684,4774, # 4896
+5685,5686,5687,3536, 957,3426,2547,2033,1931,2941,2467, 870,2019,3659,1746,2780, # 4912
+2781,2439,2468,5688,3930,5689,3789,3130,3790,3537,3427,3791,5690,1179,3086,5691, # 4928
+3187,2378,4416,3792,2548,3188,3131,2749,4143,5692,3428,1556,2549,2297, 977,2901, # 4944
+2034,4144,1205,3429,5693,1765,3430,3189,2125,1271, 714,1689,4775,3538,5694,2333, # 4960
+3931, 533,4417,3660,2184, 617,5695,2469,3340,3539,2315,5696,5697,3190,5698,5699, # 4976
+3932,1988, 618, 427,2651,3540,3431,5700,5701,1244,1690,5702,2819,4418,4776,5703, # 4992
+3541,4777,5704,2284,1576, 473,3661,4419,3432, 972,5705,3662,5706,3087,5707,5708, # 5008
+4778,4779,5709,3793,4145,4146,5710, 153,4780, 356,5711,1892,2902,4420,2144, 408, # 5024
+ 803,2357,5712,3933,5713,4421,1646,2578,2518,4781,4782,3934,5714,3935,4422,5715, # 5040
+2416,3433, 752,5716,5717,1962,3341,2979,5718, 746,3030,2470,4783,4423,3794, 698, # 5056
+4784,1893,4424,3663,2550,4785,3664,3936,5719,3191,3434,5720,1824,1302,4147,2715, # 5072
+3937,1974,4425,5721,4426,3192, 823,1303,1288,1236,2861,3542,4148,3435, 774,3938, # 5088
+5722,1581,4786,1304,2862,3939,4787,5723,2440,2162,1083,3268,4427,4149,4428, 344, # 5104
+1173, 288,2316, 454,1683,5724,5725,1461,4788,4150,2597,5726,5727,4789, 985, 894, # 5120
+5728,3436,3193,5729,1914,2942,3795,1989,5730,2111,1975,5731,4151,5732,2579,1194, # 5136
+ 425,5733,4790,3194,1245,3796,4429,5734,5735,2863,5736, 636,4791,1856,3940, 760, # 5152
+1800,5737,4430,2212,1508,4792,4152,1894,1684,2298,5738,5739,4793,4431,4432,2213, # 5168
+ 479,5740,5741, 832,5742,4153,2496,5743,2980,2497,3797, 990,3132, 627,1815,2652, # 5184
+4433,1582,4434,2126,2112,3543,4794,5744, 799,4435,3195,5745,4795,2113,1737,3031, # 5200
+1018, 543, 754,4436,3342,1676,4796,4797,4154,4798,1489,5746,3544,5747,2624,2903, # 5216
+4155,5748,5749,2981,5750,5751,5752,5753,3196,4799,4800,2185,1722,5754,3269,3270, # 5232
+1843,3665,1715, 481, 365,1976,1857,5755,5756,1963,2498,4801,5757,2127,3666,3271, # 5248
+ 433,1895,2064,2076,5758, 602,2750,5759,5760,5761,5762,5763,3032,1628,3437,5764, # 5264
+3197,4802,4156,2904,4803,2519,5765,2551,2782,5766,5767,5768,3343,4804,2905,5769, # 5280
+4805,5770,2864,4806,4807,1221,2982,4157,2520,5771,5772,5773,1868,1990,5774,5775, # 5296
+5776,1896,5777,5778,4808,1897,4158, 318,5779,2095,4159,4437,5780,5781, 485,5782, # 5312
+ 938,3941, 553,2680, 116,5783,3942,3667,5784,3545,2681,2783,3438,3344,2820,5785, # 5328
+3668,2943,4160,1747,2944,2983,5786,5787, 207,5788,4809,5789,4810,2521,5790,3033, # 5344
+ 890,3669,3943,5791,1878,3798,3439,5792,2186,2358,3440,1652,5793,5794,5795, 941, # 5360
+2299, 208,3546,4161,2020, 330,4438,3944,2906,2499,3799,4439,4811,5796,5797,5798, # 5376  #last 512
+#Everything below is of no interest for detection purpose
+2522,1613,4812,5799,3345,3945,2523,5800,4162,5801,1637,4163,2471,4813,3946,5802, # 5392
+2500,3034,3800,5803,5804,2195,4814,5805,2163,5806,5807,5808,5809,5810,5811,5812, # 5408
+5813,5814,5815,5816,5817,5818,5819,5820,5821,5822,5823,5824,5825,5826,5827,5828, # 5424
+5829,5830,5831,5832,5833,5834,5835,5836,5837,5838,5839,5840,5841,5842,5843,5844, # 5440
+5845,5846,5847,5848,5849,5850,5851,5852,5853,5854,5855,5856,5857,5858,5859,5860, # 5456
+5861,5862,5863,5864,5865,5866,5867,5868,5869,5870,5871,5872,5873,5874,5875,5876, # 5472
+5877,5878,5879,5880,5881,5882,5883,5884,5885,5886,5887,5888,5889,5890,5891,5892, # 5488
+5893,5894,5895,5896,5897,5898,5899,5900,5901,5902,5903,5904,5905,5906,5907,5908, # 5504
+5909,5910,5911,5912,5913,5914,5915,5916,5917,5918,5919,5920,5921,5922,5923,5924, # 5520
+5925,5926,5927,5928,5929,5930,5931,5932,5933,5934,5935,5936,5937,5938,5939,5940, # 5536
+5941,5942,5943,5944,5945,5946,5947,5948,5949,5950,5951,5952,5953,5954,5955,5956, # 5552
+5957,5958,5959,5960,5961,5962,5963,5964,5965,5966,5967,5968,5969,5970,5971,5972, # 5568
+5973,5974,5975,5976,5977,5978,5979,5980,5981,5982,5983,5984,5985,5986,5987,5988, # 5584
+5989,5990,5991,5992,5993,5994,5995,5996,5997,5998,5999,6000,6001,6002,6003,6004, # 5600
+6005,6006,6007,6008,6009,6010,6011,6012,6013,6014,6015,6016,6017,6018,6019,6020, # 5616
+6021,6022,6023,6024,6025,6026,6027,6028,6029,6030,6031,6032,6033,6034,6035,6036, # 5632
+6037,6038,6039,6040,6041,6042,6043,6044,6045,6046,6047,6048,6049,6050,6051,6052, # 5648
+6053,6054,6055,6056,6057,6058,6059,6060,6061,6062,6063,6064,6065,6066,6067,6068, # 5664
+6069,6070,6071,6072,6073,6074,6075,6076,6077,6078,6079,6080,6081,6082,6083,6084, # 5680
+6085,6086,6087,6088,6089,6090,6091,6092,6093,6094,6095,6096,6097,6098,6099,6100, # 5696
+6101,6102,6103,6104,6105,6106,6107,6108,6109,6110,6111,6112,6113,6114,6115,6116, # 5712
+6117,6118,6119,6120,6121,6122,6123,6124,6125,6126,6127,6128,6129,6130,6131,6132, # 5728
+6133,6134,6135,6136,6137,6138,6139,6140,6141,6142,6143,6144,6145,6146,6147,6148, # 5744
+6149,6150,6151,6152,6153,6154,6155,6156,6157,6158,6159,6160,6161,6162,6163,6164, # 5760
+6165,6166,6167,6168,6169,6170,6171,6172,6173,6174,6175,6176,6177,6178,6179,6180, # 5776
+6181,6182,6183,6184,6185,6186,6187,6188,6189,6190,6191,6192,6193,6194,6195,6196, # 5792
+6197,6198,6199,6200,6201,6202,6203,6204,6205,6206,6207,6208,6209,6210,6211,6212, # 5808
+6213,6214,6215,6216,6217,6218,6219,6220,6221,6222,6223,3670,6224,6225,6226,6227, # 5824
+6228,6229,6230,6231,6232,6233,6234,6235,6236,6237,6238,6239,6240,6241,6242,6243, # 5840
+6244,6245,6246,6247,6248,6249,6250,6251,6252,6253,6254,6255,6256,6257,6258,6259, # 5856
+6260,6261,6262,6263,6264,6265,6266,6267,6268,6269,6270,6271,6272,6273,6274,6275, # 5872
+6276,6277,6278,6279,6280,6281,6282,6283,6284,6285,4815,6286,6287,6288,6289,6290, # 5888
+6291,6292,4816,6293,6294,6295,6296,6297,6298,6299,6300,6301,6302,6303,6304,6305, # 5904
+6306,6307,6308,6309,6310,6311,4817,4818,6312,6313,6314,6315,6316,6317,6318,4819, # 5920
+6319,6320,6321,6322,6323,6324,6325,6326,6327,6328,6329,6330,6331,6332,6333,6334, # 5936
+6335,6336,6337,4820,6338,6339,6340,6341,6342,6343,6344,6345,6346,6347,6348,6349, # 5952
+6350,6351,6352,6353,6354,6355,6356,6357,6358,6359,6360,6361,6362,6363,6364,6365, # 5968
+6366,6367,6368,6369,6370,6371,6372,6373,6374,6375,6376,6377,6378,6379,6380,6381, # 5984
+6382,6383,6384,6385,6386,6387,6388,6389,6390,6391,6392,6393,6394,6395,6396,6397, # 6000
+6398,6399,6400,6401,6402,6403,6404,6405,6406,6407,6408,6409,6410,3441,6411,6412, # 6016
+6413,6414,6415,6416,6417,6418,6419,6420,6421,6422,6423,6424,6425,4440,6426,6427, # 6032
+6428,6429,6430,6431,6432,6433,6434,6435,6436,6437,6438,6439,6440,6441,6442,6443, # 6048
+6444,6445,6446,6447,6448,6449,6450,6451,6452,6453,6454,4821,6455,6456,6457,6458, # 6064
+6459,6460,6461,6462,6463,6464,6465,6466,6467,6468,6469,6470,6471,6472,6473,6474, # 6080
+6475,6476,6477,3947,3948,6478,6479,6480,6481,3272,4441,6482,6483,6484,6485,4442, # 6096
+6486,6487,6488,6489,6490,6491,6492,6493,6494,6495,6496,4822,6497,6498,6499,6500, # 6112
+6501,6502,6503,6504,6505,6506,6507,6508,6509,6510,6511,6512,6513,6514,6515,6516, # 6128
+6517,6518,6519,6520,6521,6522,6523,6524,6525,6526,6527,6528,6529,6530,6531,6532, # 6144
+6533,6534,6535,6536,6537,6538,6539,6540,6541,6542,6543,6544,6545,6546,6547,6548, # 6160
+6549,6550,6551,6552,6553,6554,6555,6556,2784,6557,4823,6558,6559,6560,6561,6562, # 6176
+6563,6564,6565,6566,6567,6568,6569,3949,6570,6571,6572,4824,6573,6574,6575,6576, # 6192
+6577,6578,6579,6580,6581,6582,6583,4825,6584,6585,6586,3950,2785,6587,6588,6589, # 6208
+6590,6591,6592,6593,6594,6595,6596,6597,6598,6599,6600,6601,6602,6603,6604,6605, # 6224
+6606,6607,6608,6609,6610,6611,6612,4826,6613,6614,6615,4827,6616,6617,6618,6619, # 6240
+6620,6621,6622,6623,6624,6625,4164,6626,6627,6628,6629,6630,6631,6632,6633,6634, # 6256
+3547,6635,4828,6636,6637,6638,6639,6640,6641,6642,3951,2984,6643,6644,6645,6646, # 6272
+6647,6648,6649,4165,6650,4829,6651,6652,4830,6653,6654,6655,6656,6657,6658,6659, # 6288
+6660,6661,6662,4831,6663,6664,6665,6666,6667,6668,6669,6670,6671,4166,6672,4832, # 6304
+3952,6673,6674,6675,6676,4833,6677,6678,6679,4167,6680,6681,6682,3198,6683,6684, # 6320
+6685,6686,6687,6688,6689,6690,6691,6692,6693,6694,6695,6696,6697,4834,6698,6699, # 6336
+6700,6701,6702,6703,6704,6705,6706,6707,6708,6709,6710,6711,6712,6713,6714,6715, # 6352
+6716,6717,6718,6719,6720,6721,6722,6723,6724,6725,6726,6727,6728,6729,6730,6731, # 6368
+6732,6733,6734,4443,6735,6736,6737,6738,6739,6740,6741,6742,6743,6744,6745,4444, # 6384
+6746,6747,6748,6749,6750,6751,6752,6753,6754,6755,6756,6757,6758,6759,6760,6761, # 6400
+6762,6763,6764,6765,6766,6767,6768,6769,6770,6771,6772,6773,6774,6775,6776,6777, # 6416
+6778,6779,6780,6781,4168,6782,6783,3442,6784,6785,6786,6787,6788,6789,6790,6791, # 6432
+4169,6792,6793,6794,6795,6796,6797,6798,6799,6800,6801,6802,6803,6804,6805,6806, # 6448
+6807,6808,6809,6810,6811,4835,6812,6813,6814,4445,6815,6816,4446,6817,6818,6819, # 6464
+6820,6821,6822,6823,6824,6825,6826,6827,6828,6829,6830,6831,6832,6833,6834,6835, # 6480
+3548,6836,6837,6838,6839,6840,6841,6842,6843,6844,6845,6846,4836,6847,6848,6849, # 6496
+6850,6851,6852,6853,6854,3953,6855,6856,6857,6858,6859,6860,6861,6862,6863,6864, # 6512
+6865,6866,6867,6868,6869,6870,6871,6872,6873,6874,6875,6876,6877,3199,6878,6879, # 6528
+6880,6881,6882,4447,6883,6884,6885,6886,6887,6888,6889,6890,6891,6892,6893,6894, # 6544
+6895,6896,6897,6898,6899,6900,6901,6902,6903,6904,4170,6905,6906,6907,6908,6909, # 6560
+6910,6911,6912,6913,6914,6915,6916,6917,6918,6919,6920,6921,6922,6923,6924,6925, # 6576
+6926,6927,4837,6928,6929,6930,6931,6932,6933,6934,6935,6936,3346,6937,6938,4838, # 6592
+6939,6940,6941,4448,6942,6943,6944,6945,6946,4449,6947,6948,6949,6950,6951,6952, # 6608
+6953,6954,6955,6956,6957,6958,6959,6960,6961,6962,6963,6964,6965,6966,6967,6968, # 6624
+6969,6970,6971,6972,6973,6974,6975,6976,6977,6978,6979,6980,6981,6982,6983,6984, # 6640
+6985,6986,6987,6988,6989,6990,6991,6992,6993,6994,3671,6995,6996,6997,6998,4839, # 6656
+6999,7000,7001,7002,3549,7003,7004,7005,7006,7007,7008,7009,7010,7011,7012,7013, # 6672
+7014,7015,7016,7017,7018,7019,7020,7021,7022,7023,7024,7025,7026,7027,7028,7029, # 6688
+7030,4840,7031,7032,7033,7034,7035,7036,7037,7038,4841,7039,7040,7041,7042,7043, # 6704
+7044,7045,7046,7047,7048,7049,7050,7051,7052,7053,7054,7055,7056,7057,7058,7059, # 6720
+7060,7061,7062,7063,7064,7065,7066,7067,7068,7069,7070,2985,7071,7072,7073,7074, # 6736
+7075,7076,7077,7078,7079,7080,4842,7081,7082,7083,7084,7085,7086,7087,7088,7089, # 6752
+7090,7091,7092,7093,7094,7095,7096,7097,7098,7099,7100,7101,7102,7103,7104,7105, # 6768
+7106,7107,7108,7109,7110,7111,7112,7113,7114,7115,7116,7117,7118,4450,7119,7120, # 6784
+7121,7122,7123,7124,7125,7126,7127,7128,7129,7130,7131,7132,7133,7134,7135,7136, # 6800
+7137,7138,7139,7140,7141,7142,7143,4843,7144,7145,7146,7147,7148,7149,7150,7151, # 6816
+7152,7153,7154,7155,7156,7157,7158,7159,7160,7161,7162,7163,7164,7165,7166,7167, # 6832
+7168,7169,7170,7171,7172,7173,7174,7175,7176,7177,7178,7179,7180,7181,7182,7183, # 6848
+7184,7185,7186,7187,7188,4171,4172,7189,7190,7191,7192,7193,7194,7195,7196,7197, # 6864
+7198,7199,7200,7201,7202,7203,7204,7205,7206,7207,7208,7209,7210,7211,7212,7213, # 6880
+7214,7215,7216,7217,7218,7219,7220,7221,7222,7223,7224,7225,7226,7227,7228,7229, # 6896
+7230,7231,7232,7233,7234,7235,7236,7237,7238,7239,7240,7241,7242,7243,7244,7245, # 6912
+7246,7247,7248,7249,7250,7251,7252,7253,7254,7255,7256,7257,7258,7259,7260,7261, # 6928
+7262,7263,7264,7265,7266,7267,7268,7269,7270,7271,7272,7273,7274,7275,7276,7277, # 6944
+7278,7279,7280,7281,7282,7283,7284,7285,7286,7287,7288,7289,7290,7291,7292,7293, # 6960
+7294,7295,7296,4844,7297,7298,7299,7300,7301,7302,7303,7304,7305,7306,7307,7308, # 6976
+7309,7310,7311,7312,7313,7314,7315,7316,4451,7317,7318,7319,7320,7321,7322,7323, # 6992
+7324,7325,7326,7327,7328,7329,7330,7331,7332,7333,7334,7335,7336,7337,7338,7339, # 7008
+7340,7341,7342,7343,7344,7345,7346,7347,7348,7349,7350,7351,7352,7353,4173,7354, # 7024
+7355,4845,7356,7357,7358,7359,7360,7361,7362,7363,7364,7365,7366,7367,7368,7369, # 7040
+7370,7371,7372,7373,7374,7375,7376,7377,7378,7379,7380,7381,7382,7383,7384,7385, # 7056
+7386,7387,7388,4846,7389,7390,7391,7392,7393,7394,7395,7396,7397,7398,7399,7400, # 7072
+7401,7402,7403,7404,7405,3672,7406,7407,7408,7409,7410,7411,7412,7413,7414,7415, # 7088
+7416,7417,7418,7419,7420,7421,7422,7423,7424,7425,7426,7427,7428,7429,7430,7431, # 7104
+7432,7433,7434,7435,7436,7437,7438,7439,7440,7441,7442,7443,7444,7445,7446,7447, # 7120
+7448,7449,7450,7451,7452,7453,4452,7454,3200,7455,7456,7457,7458,7459,7460,7461, # 7136
+7462,7463,7464,7465,7466,7467,7468,7469,7470,7471,7472,7473,7474,4847,7475,7476, # 7152
+7477,3133,7478,7479,7480,7481,7482,7483,7484,7485,7486,7487,7488,7489,7490,7491, # 7168
+7492,7493,7494,7495,7496,7497,7498,7499,7500,7501,7502,3347,7503,7504,7505,7506, # 7184
+7507,7508,7509,7510,7511,7512,7513,7514,7515,7516,7517,7518,7519,7520,7521,4848, # 7200
+7522,7523,7524,7525,7526,7527,7528,7529,7530,7531,7532,7533,7534,7535,7536,7537, # 7216
+7538,7539,7540,7541,7542,7543,7544,7545,7546,7547,7548,7549,3801,4849,7550,7551, # 7232
+7552,7553,7554,7555,7556,7557,7558,7559,7560,7561,7562,7563,7564,7565,7566,7567, # 7248
+7568,7569,3035,7570,7571,7572,7573,7574,7575,7576,7577,7578,7579,7580,7581,7582, # 7264
+7583,7584,7585,7586,7587,7588,7589,7590,7591,7592,7593,7594,7595,7596,7597,7598, # 7280
+7599,7600,7601,7602,7603,7604,7605,7606,7607,7608,7609,7610,7611,7612,7613,7614, # 7296
+7615,7616,4850,7617,7618,3802,7619,7620,7621,7622,7623,7624,7625,7626,7627,7628, # 7312
+7629,7630,7631,7632,4851,7633,7634,7635,7636,7637,7638,7639,7640,7641,7642,7643, # 7328
+7644,7645,7646,7647,7648,7649,7650,7651,7652,7653,7654,7655,7656,7657,7658,7659, # 7344
+7660,7661,7662,7663,7664,7665,7666,7667,7668,7669,7670,4453,7671,7672,7673,7674, # 7360
+7675,7676,7677,7678,7679,7680,7681,7682,7683,7684,7685,7686,7687,7688,7689,7690, # 7376
+7691,7692,7693,7694,7695,7696,7697,3443,7698,7699,7700,7701,7702,4454,7703,7704, # 7392
+7705,7706,7707,7708,7709,7710,7711,7712,7713,2472,7714,7715,7716,7717,7718,7719, # 7408
+7720,7721,7722,7723,7724,7725,7726,7727,7728,7729,7730,7731,3954,7732,7733,7734, # 7424
+7735,7736,7737,7738,7739,7740,7741,7742,7743,7744,7745,7746,7747,7748,7749,7750, # 7440
+3134,7751,7752,4852,7753,7754,7755,4853,7756,7757,7758,7759,7760,4174,7761,7762, # 7456
+7763,7764,7765,7766,7767,7768,7769,7770,7771,7772,7773,7774,7775,7776,7777,7778, # 7472
+7779,7780,7781,7782,7783,7784,7785,7786,7787,7788,7789,7790,7791,7792,7793,7794, # 7488
+7795,7796,7797,7798,7799,7800,7801,7802,7803,7804,7805,4854,7806,7807,7808,7809, # 7504
+7810,7811,7812,7813,7814,7815,7816,7817,7818,7819,7820,7821,7822,7823,7824,7825, # 7520
+4855,7826,7827,7828,7829,7830,7831,7832,7833,7834,7835,7836,7837,7838,7839,7840, # 7536
+7841,7842,7843,7844,7845,7846,7847,3955,7848,7849,7850,7851,7852,7853,7854,7855, # 7552
+7856,7857,7858,7859,7860,3444,7861,7862,7863,7864,7865,7866,7867,7868,7869,7870, # 7568
+7871,7872,7873,7874,7875,7876,7877,7878,7879,7880,7881,7882,7883,7884,7885,7886, # 7584
+7887,7888,7889,7890,7891,4175,7892,7893,7894,7895,7896,4856,4857,7897,7898,7899, # 7600
+7900,2598,7901,7902,7903,7904,7905,7906,7907,7908,4455,7909,7910,7911,7912,7913, # 7616
+7914,3201,7915,7916,7917,7918,7919,7920,7921,4858,7922,7923,7924,7925,7926,7927, # 7632
+7928,7929,7930,7931,7932,7933,7934,7935,7936,7937,7938,7939,7940,7941,7942,7943, # 7648
+7944,7945,7946,7947,7948,7949,7950,7951,7952,7953,7954,7955,7956,7957,7958,7959, # 7664
+7960,7961,7962,7963,7964,7965,7966,7967,7968,7969,7970,7971,7972,7973,7974,7975, # 7680
+7976,7977,7978,7979,7980,7981,4859,7982,7983,7984,7985,7986,7987,7988,7989,7990, # 7696
+7991,7992,7993,7994,7995,7996,4860,7997,7998,7999,8000,8001,8002,8003,8004,8005, # 7712
+8006,8007,8008,8009,8010,8011,8012,8013,8014,8015,8016,4176,8017,8018,8019,8020, # 7728
+8021,8022,8023,4861,8024,8025,8026,8027,8028,8029,8030,8031,8032,8033,8034,8035, # 7744
+8036,4862,4456,8037,8038,8039,8040,4863,8041,8042,8043,8044,8045,8046,8047,8048, # 7760
+8049,8050,8051,8052,8053,8054,8055,8056,8057,8058,8059,8060,8061,8062,8063,8064, # 7776
+8065,8066,8067,8068,8069,8070,8071,8072,8073,8074,8075,8076,8077,8078,8079,8080, # 7792
+8081,8082,8083,8084,8085,8086,8087,8088,8089,8090,8091,8092,8093,8094,8095,8096, # 7808
+8097,8098,8099,4864,4177,8100,8101,8102,8103,8104,8105,8106,8107,8108,8109,8110, # 7824
+8111,8112,8113,8114,8115,8116,8117,8118,8119,8120,4178,8121,8122,8123,8124,8125, # 7840
+8126,8127,8128,8129,8130,8131,8132,8133,8134,8135,8136,8137,8138,8139,8140,8141, # 7856
+8142,8143,8144,8145,4865,4866,8146,8147,8148,8149,8150,8151,8152,8153,8154,8155, # 7872
+8156,8157,8158,8159,8160,8161,8162,8163,8164,8165,4179,8166,8167,8168,8169,8170, # 7888
+8171,8172,8173,8174,8175,8176,8177,8178,8179,8180,8181,4457,8182,8183,8184,8185, # 7904
+8186,8187,8188,8189,8190,8191,8192,8193,8194,8195,8196,8197,8198,8199,8200,8201, # 7920
+8202,8203,8204,8205,8206,8207,8208,8209,8210,8211,8212,8213,8214,8215,8216,8217, # 7936
+8218,8219,8220,8221,8222,8223,8224,8225,8226,8227,8228,8229,8230,8231,8232,8233, # 7952
+8234,8235,8236,8237,8238,8239,8240,8241,8242,8243,8244,8245,8246,8247,8248,8249, # 7968
+8250,8251,8252,8253,8254,8255,8256,3445,8257,8258,8259,8260,8261,8262,4458,8263, # 7984
+8264,8265,8266,8267,8268,8269,8270,8271,8272,4459,8273,8274,8275,8276,3550,8277, # 8000
+8278,8279,8280,8281,8282,8283,8284,8285,8286,8287,8288,8289,4460,8290,8291,8292, # 8016
+8293,8294,8295,8296,8297,8298,8299,8300,8301,8302,8303,8304,8305,8306,8307,4867, # 8032
+8308,8309,8310,8311,8312,3551,8313,8314,8315,8316,8317,8318,8319,8320,8321,8322, # 8048
+8323,8324,8325,8326,4868,8327,8328,8329,8330,8331,8332,8333,8334,8335,8336,8337, # 8064
+8338,8339,8340,8341,8342,8343,8344,8345,8346,8347,8348,8349,8350,8351,8352,8353, # 8080
+8354,8355,8356,8357,8358,8359,8360,8361,8362,8363,4869,4461,8364,8365,8366,8367, # 8096
+8368,8369,8370,4870,8371,8372,8373,8374,8375,8376,8377,8378,8379,8380,8381,8382, # 8112
+8383,8384,8385,8386,8387,8388,8389,8390,8391,8392,8393,8394,8395,8396,8397,8398, # 8128
+8399,8400,8401,8402,8403,8404,8405,8406,8407,8408,8409,8410,4871,8411,8412,8413, # 8144
+8414,8415,8416,8417,8418,8419,8420,8421,8422,4462,8423,8424,8425,8426,8427,8428, # 8160
+8429,8430,8431,8432,8433,2986,8434,8435,8436,8437,8438,8439,8440,8441,8442,8443, # 8176
+8444,8445,8446,8447,8448,8449,8450,8451,8452,8453,8454,8455,8456,8457,8458,8459, # 8192
+8460,8461,8462,8463,8464,8465,8466,8467,8468,8469,8470,8471,8472,8473,8474,8475, # 8208
+8476,8477,8478,4180,8479,8480,8481,8482,8483,8484,8485,8486,8487,8488,8489,8490, # 8224
+8491,8492,8493,8494,8495,8496,8497,8498,8499,8500,8501,8502,8503,8504,8505,8506, # 8240
+8507,8508,8509,8510,8511,8512,8513,8514,8515,8516,8517,8518,8519,8520,8521,8522, # 8256
+8523,8524,8525,8526,8527,8528,8529,8530,8531,8532,8533,8534,8535,8536,8537,8538, # 8272
+8539,8540,8541,8542,8543,8544,8545,8546,8547,8548,8549,8550,8551,8552,8553,8554, # 8288
+8555,8556,8557,8558,8559,8560,8561,8562,8563,8564,4872,8565,8566,8567,8568,8569, # 8304
+8570,8571,8572,8573,4873,8574,8575,8576,8577,8578,8579,8580,8581,8582,8583,8584, # 8320
+8585,8586,8587,8588,8589,8590,8591,8592,8593,8594,8595,8596,8597,8598,8599,8600, # 8336
+8601,8602,8603,8604,8605,3803,8606,8607,8608,8609,8610,8611,8612,8613,4874,3804, # 8352
+8614,8615,8616,8617,8618,8619,8620,8621,3956,8622,8623,8624,8625,8626,8627,8628, # 8368
+8629,8630,8631,8632,8633,8634,8635,8636,8637,8638,2865,8639,8640,8641,8642,8643, # 8384
+8644,8645,8646,8647,8648,8649,8650,8651,8652,8653,8654,8655,8656,4463,8657,8658, # 8400
+8659,4875,4876,8660,8661,8662,8663,8664,8665,8666,8667,8668,8669,8670,8671,8672, # 8416
+8673,8674,8675,8676,8677,8678,8679,8680,8681,4464,8682,8683,8684,8685,8686,8687, # 8432
+8688,8689,8690,8691,8692,8693,8694,8695,8696,8697,8698,8699,8700,8701,8702,8703, # 8448
+8704,8705,8706,8707,8708,8709,2261,8710,8711,8712,8713,8714,8715,8716,8717,8718, # 8464
+8719,8720,8721,8722,8723,8724,8725,8726,8727,8728,8729,8730,8731,8732,8733,4181, # 8480
+8734,8735,8736,8737,8738,8739,8740,8741,8742,8743,8744,8745,8746,8747,8748,8749, # 8496
+8750,8751,8752,8753,8754,8755,8756,8757,8758,8759,8760,8761,8762,8763,4877,8764, # 8512
+8765,8766,8767,8768,8769,8770,8771,8772,8773,8774,8775,8776,8777,8778,8779,8780, # 8528
+8781,8782,8783,8784,8785,8786,8787,8788,4878,8789,4879,8790,8791,8792,4880,8793, # 8544
+8794,8795,8796,8797,8798,8799,8800,8801,4881,8802,8803,8804,8805,8806,8807,8808, # 8560
+8809,8810,8811,8812,8813,8814,8815,3957,8816,8817,8818,8819,8820,8821,8822,8823, # 8576
+8824,8825,8826,8827,8828,8829,8830,8831,8832,8833,8834,8835,8836,8837,8838,8839, # 8592
+8840,8841,8842,8843,8844,8845,8846,8847,4882,8848,8849,8850,8851,8852,8853,8854, # 8608
+8855,8856,8857,8858,8859,8860,8861,8862,8863,8864,8865,8866,8867,8868,8869,8870, # 8624
+8871,8872,8873,8874,8875,8876,8877,8878,8879,8880,8881,8882,8883,8884,3202,8885, # 8640
+8886,8887,8888,8889,8890,8891,8892,8893,8894,8895,8896,8897,8898,8899,8900,8901, # 8656
+8902,8903,8904,8905,8906,8907,8908,8909,8910,8911,8912,8913,8914,8915,8916,8917, # 8672
+8918,8919,8920,8921,8922,8923,8924,4465,8925,8926,8927,8928,8929,8930,8931,8932, # 8688
+4883,8933,8934,8935,8936,8937,8938,8939,8940,8941,8942,8943,2214,8944,8945,8946, # 8704
+8947,8948,8949,8950,8951,8952,8953,8954,8955,8956,8957,8958,8959,8960,8961,8962, # 8720
+8963,8964,8965,4884,8966,8967,8968,8969,8970,8971,8972,8973,8974,8975,8976,8977, # 8736
+8978,8979,8980,8981,8982,8983,8984,8985,8986,8987,8988,8989,8990,8991,8992,4885, # 8752
+8993,8994,8995,8996,8997,8998,8999,9000,9001,9002,9003,9004,9005,9006,9007,9008, # 8768
+9009,9010,9011,9012,9013,9014,9015,9016,9017,9018,9019,9020,9021,4182,9022,9023, # 8784
+9024,9025,9026,9027,9028,9029,9030,9031,9032,9033,9034,9035,9036,9037,9038,9039, # 8800
+9040,9041,9042,9043,9044,9045,9046,9047,9048,9049,9050,9051,9052,9053,9054,9055, # 8816
+9056,9057,9058,9059,9060,9061,9062,9063,4886,9064,9065,9066,9067,9068,9069,4887, # 8832
+9070,9071,9072,9073,9074,9075,9076,9077,9078,9079,9080,9081,9082,9083,9084,9085, # 8848
+9086,9087,9088,9089,9090,9091,9092,9093,9094,9095,9096,9097,9098,9099,9100,9101, # 8864
+9102,9103,9104,9105,9106,9107,9108,9109,9110,9111,9112,9113,9114,9115,9116,9117, # 8880
+9118,9119,9120,9121,9122,9123,9124,9125,9126,9127,9128,9129,9130,9131,9132,9133, # 8896
+9134,9135,9136,9137,9138,9139,9140,9141,3958,9142,9143,9144,9145,9146,9147,9148, # 8912
+9149,9150,9151,4888,9152,9153,9154,9155,9156,9157,9158,9159,9160,9161,9162,9163, # 8928
+9164,9165,9166,9167,9168,9169,9170,9171,9172,9173,9174,9175,4889,9176,9177,9178, # 8944
+9179,9180,9181,9182,9183,9184,9185,9186,9187,9188,9189,9190,9191,9192,9193,9194, # 8960
+9195,9196,9197,9198,9199,9200,9201,9202,9203,4890,9204,9205,9206,9207,9208,9209, # 8976
+9210,9211,9212,9213,9214,9215,9216,9217,9218,9219,9220,9221,9222,4466,9223,9224, # 8992
+9225,9226,9227,9228,9229,9230,9231,9232,9233,9234,9235,9236,9237,9238,9239,9240, # 9008
+9241,9242,9243,9244,9245,4891,9246,9247,9248,9249,9250,9251,9252,9253,9254,9255, # 9024
+9256,9257,4892,9258,9259,9260,9261,4893,4894,9262,9263,9264,9265,9266,9267,9268, # 9040
+9269,9270,9271,9272,9273,4467,9274,9275,9276,9277,9278,9279,9280,9281,9282,9283, # 9056
+9284,9285,3673,9286,9287,9288,9289,9290,9291,9292,9293,9294,9295,9296,9297,9298, # 9072
+9299,9300,9301,9302,9303,9304,9305,9306,9307,9308,9309,9310,9311,9312,9313,9314, # 9088
+9315,9316,9317,9318,9319,9320,9321,9322,4895,9323,9324,9325,9326,9327,9328,9329, # 9104
+9330,9331,9332,9333,9334,9335,9336,9337,9338,9339,9340,9341,9342,9343,9344,9345, # 9120
+9346,9347,4468,9348,9349,9350,9351,9352,9353,9354,9355,9356,9357,9358,9359,9360, # 9136
+9361,9362,9363,9364,9365,9366,9367,9368,9369,9370,9371,9372,9373,4896,9374,4469, # 9152
+9375,9376,9377,9378,9379,4897,9380,9381,9382,9383,9384,9385,9386,9387,9388,9389, # 9168
+9390,9391,9392,9393,9394,9395,9396,9397,9398,9399,9400,9401,9402,9403,9404,9405, # 9184
+9406,4470,9407,2751,9408,9409,3674,3552,9410,9411,9412,9413,9414,9415,9416,9417, # 9200
+9418,9419,9420,9421,4898,9422,9423,9424,9425,9426,9427,9428,9429,3959,9430,9431, # 9216
+9432,9433,9434,9435,9436,4471,9437,9438,9439,9440,9441,9442,9443,9444,9445,9446, # 9232
+9447,9448,9449,9450,3348,9451,9452,9453,9454,9455,9456,9457,9458,9459,9460,9461, # 9248
+9462,9463,9464,9465,9466,9467,9468,9469,9470,9471,9472,4899,9473,9474,9475,9476, # 9264
+9477,4900,9478,9479,9480,9481,9482,9483,9484,9485,9486,9487,9488,3349,9489,9490, # 9280
+9491,9492,9493,9494,9495,9496,9497,9498,9499,9500,9501,9502,9503,9504,9505,9506, # 9296
+9507,9508,9509,9510,9511,9512,9513,9514,9515,9516,9517,9518,9519,9520,4901,9521, # 9312
+9522,9523,9524,9525,9526,4902,9527,9528,9529,9530,9531,9532,9533,9534,9535,9536, # 9328
+9537,9538,9539,9540,9541,9542,9543,9544,9545,9546,9547,9548,9549,9550,9551,9552, # 9344
+9553,9554,9555,9556,9557,9558,9559,9560,9561,9562,9563,9564,9565,9566,9567,9568, # 9360
+9569,9570,9571,9572,9573,9574,9575,9576,9577,9578,9579,9580,9581,9582,9583,9584, # 9376
+3805,9585,9586,9587,9588,9589,9590,9591,9592,9593,9594,9595,9596,9597,9598,9599, # 9392
+9600,9601,9602,4903,9603,9604,9605,9606,9607,4904,9608,9609,9610,9611,9612,9613, # 9408
+9614,4905,9615,9616,9617,9618,9619,9620,9621,9622,9623,9624,9625,9626,9627,9628, # 9424
+9629,9630,9631,9632,4906,9633,9634,9635,9636,9637,9638,9639,9640,9641,9642,9643, # 9440
+4907,9644,9645,9646,9647,9648,9649,9650,9651,9652,9653,9654,9655,9656,9657,9658, # 9456
+9659,9660,9661,9662,9663,9664,9665,9666,9667,9668,9669,9670,9671,9672,4183,9673, # 9472
+9674,9675,9676,9677,4908,9678,9679,9680,9681,4909,9682,9683,9684,9685,9686,9687, # 9488
+9688,9689,9690,4910,9691,9692,9693,3675,9694,9695,9696,2945,9697,9698,9699,9700, # 9504
+9701,9702,9703,9704,9705,4911,9706,9707,9708,9709,9710,9711,9712,9713,9714,9715, # 9520
+9716,9717,9718,9719,9720,9721,9722,9723,9724,9725,9726,9727,9728,9729,9730,9731, # 9536
+9732,9733,9734,9735,4912,9736,9737,9738,9739,9740,4913,9741,9742,9743,9744,9745, # 9552
+9746,9747,9748,9749,9750,9751,9752,9753,9754,9755,9756,9757,9758,4914,9759,9760, # 9568
+9761,9762,9763,9764,9765,9766,9767,9768,9769,9770,9771,9772,9773,9774,9775,9776, # 9584
+9777,9778,9779,9780,9781,9782,4915,9783,9784,9785,9786,9787,9788,9789,9790,9791, # 9600
+9792,9793,4916,9794,9795,9796,9797,9798,9799,9800,9801,9802,9803,9804,9805,9806, # 9616
+9807,9808,9809,9810,9811,9812,9813,9814,9815,9816,9817,9818,9819,9820,9821,9822, # 9632
+9823,9824,9825,9826,9827,9828,9829,9830,9831,9832,9833,9834,9835,9836,9837,9838, # 9648
+9839,9840,9841,9842,9843,9844,9845,9846,9847,9848,9849,9850,9851,9852,9853,9854, # 9664
+9855,9856,9857,9858,9859,9860,9861,9862,9863,9864,9865,9866,9867,9868,4917,9869, # 9680
+9870,9871,9872,9873,9874,9875,9876,9877,9878,9879,9880,9881,9882,9883,9884,9885, # 9696
+9886,9887,9888,9889,9890,9891,9892,4472,9893,9894,9895,9896,9897,3806,9898,9899, # 9712
+9900,9901,9902,9903,9904,9905,9906,9907,9908,9909,9910,9911,9912,9913,9914,4918, # 9728
+9915,9916,9917,4919,9918,9919,9920,9921,4184,9922,9923,9924,9925,9926,9927,9928, # 9744
+9929,9930,9931,9932,9933,9934,9935,9936,9937,9938,9939,9940,9941,9942,9943,9944, # 9760
+9945,9946,4920,9947,9948,9949,9950,9951,9952,9953,9954,9955,4185,9956,9957,9958, # 9776
+9959,9960,9961,9962,9963,9964,9965,4921,9966,9967,9968,4473,9969,9970,9971,9972, # 9792
+9973,9974,9975,9976,9977,4474,9978,9979,9980,9981,9982,9983,9984,9985,9986,9987, # 9808
+9988,9989,9990,9991,9992,9993,9994,9995,9996,9997,9998,9999,10000,10001,10002,10003, # 9824
+10004,10005,10006,10007,10008,10009,10010,10011,10012,10013,10014,10015,10016,10017,10018,10019, # 9840
+10020,10021,4922,10022,4923,10023,10024,10025,10026,10027,10028,10029,10030,10031,10032,10033, # 9856
+10034,10035,10036,10037,10038,10039,10040,10041,10042,10043,10044,10045,10046,10047,10048,4924, # 9872
+10049,10050,10051,10052,10053,10054,10055,10056,10057,10058,10059,10060,10061,10062,10063,10064, # 9888
+10065,10066,10067,10068,10069,10070,10071,10072,10073,10074,10075,10076,10077,10078,10079,10080, # 9904
+10081,10082,10083,10084,10085,10086,10087,4475,10088,10089,10090,10091,10092,10093,10094,10095, # 9920
+10096,10097,4476,10098,10099,10100,10101,10102,10103,10104,10105,10106,10107,10108,10109,10110, # 9936
+10111,2174,10112,10113,10114,10115,10116,10117,10118,10119,10120,10121,10122,10123,10124,10125, # 9952
+10126,10127,10128,10129,10130,10131,10132,10133,10134,10135,10136,10137,10138,10139,10140,3807, # 9968
+4186,4925,10141,10142,10143,10144,10145,10146,10147,4477,4187,10148,10149,10150,10151,10152, # 9984
+10153,4188,10154,10155,10156,10157,10158,10159,10160,10161,4926,10162,10163,10164,10165,10166, #10000
+10167,10168,10169,10170,10171,10172,10173,10174,10175,10176,10177,10178,10179,10180,10181,10182, #10016
+10183,10184,10185,10186,10187,10188,10189,10190,10191,10192,3203,10193,10194,10195,10196,10197, #10032
+10198,10199,10200,4478,10201,10202,10203,10204,4479,10205,10206,10207,10208,10209,10210,10211, #10048
+10212,10213,10214,10215,10216,10217,10218,10219,10220,10221,10222,10223,10224,10225,10226,10227, #10064
+10228,10229,10230,10231,10232,10233,10234,4927,10235,10236,10237,10238,10239,10240,10241,10242, #10080
+10243,10244,10245,10246,10247,10248,10249,10250,10251,10252,10253,10254,10255,10256,10257,10258, #10096
+10259,10260,10261,10262,10263,10264,10265,10266,10267,10268,10269,10270,10271,10272,10273,4480, #10112
+4928,4929,10274,10275,10276,10277,10278,10279,10280,10281,10282,10283,10284,10285,10286,10287, #10128
+10288,10289,10290,10291,10292,10293,10294,10295,10296,10297,10298,10299,10300,10301,10302,10303, #10144
+10304,10305,10306,10307,10308,10309,10310,10311,10312,10313,10314,10315,10316,10317,10318,10319, #10160
+10320,10321,10322,10323,10324,10325,10326,10327,10328,10329,10330,10331,10332,10333,10334,4930, #10176
+10335,10336,10337,10338,10339,10340,10341,10342,4931,10343,10344,10345,10346,10347,10348,10349, #10192
+10350,10351,10352,10353,10354,10355,3088,10356,2786,10357,10358,10359,10360,4189,10361,10362, #10208
+10363,10364,10365,10366,10367,10368,10369,10370,10371,10372,10373,10374,10375,4932,10376,10377, #10224
+10378,10379,10380,10381,10382,10383,10384,10385,10386,10387,10388,10389,10390,10391,10392,4933, #10240
+10393,10394,10395,4934,10396,10397,10398,10399,10400,10401,10402,10403,10404,10405,10406,10407, #10256
+10408,10409,10410,10411,10412,3446,10413,10414,10415,10416,10417,10418,10419,10420,10421,10422, #10272
+10423,4935,10424,10425,10426,10427,10428,10429,10430,4936,10431,10432,10433,10434,10435,10436, #10288
+10437,10438,10439,10440,10441,10442,10443,4937,10444,10445,10446,10447,4481,10448,10449,10450, #10304
+10451,10452,10453,10454,10455,10456,10457,10458,10459,10460,10461,10462,10463,10464,10465,10466, #10320
+10467,10468,10469,10470,10471,10472,10473,10474,10475,10476,10477,10478,10479,10480,10481,10482, #10336
+10483,10484,10485,10486,10487,10488,10489,10490,10491,10492,10493,10494,10495,10496,10497,10498, #10352
+10499,10500,10501,10502,10503,10504,10505,4938,10506,10507,10508,10509,10510,2552,10511,10512, #10368
+10513,10514,10515,10516,3447,10517,10518,10519,10520,10521,10522,10523,10524,10525,10526,10527, #10384
+10528,10529,10530,10531,10532,10533,10534,10535,10536,10537,10538,10539,10540,10541,10542,10543, #10400
+4482,10544,4939,10545,10546,10547,10548,10549,10550,10551,10552,10553,10554,10555,10556,10557, #10416
+10558,10559,10560,10561,10562,10563,10564,10565,10566,10567,3676,4483,10568,10569,10570,10571, #10432
+10572,3448,10573,10574,10575,10576,10577,10578,10579,10580,10581,10582,10583,10584,10585,10586, #10448
+10587,10588,10589,10590,10591,10592,10593,10594,10595,10596,10597,10598,10599,10600,10601,10602, #10464
+10603,10604,10605,10606,10607,10608,10609,10610,10611,10612,10613,10614,10615,10616,10617,10618, #10480
+10619,10620,10621,10622,10623,10624,10625,10626,10627,4484,10628,10629,10630,10631,10632,4940, #10496
+10633,10634,10635,10636,10637,10638,10639,10640,10641,10642,10643,10644,10645,10646,10647,10648, #10512
+10649,10650,10651,10652,10653,10654,10655,10656,4941,10657,10658,10659,2599,10660,10661,10662, #10528
+10663,10664,10665,10666,3089,10667,10668,10669,10670,10671,10672,10673,10674,10675,10676,10677, #10544
+10678,10679,10680,4942,10681,10682,10683,10684,10685,10686,10687,10688,10689,10690,10691,10692, #10560
+10693,10694,10695,10696,10697,4485,10698,10699,10700,10701,10702,10703,10704,4943,10705,3677, #10576
+10706,10707,10708,10709,10710,10711,10712,4944,10713,10714,10715,10716,10717,10718,10719,10720, #10592
+10721,10722,10723,10724,10725,10726,10727,10728,4945,10729,10730,10731,10732,10733,10734,10735, #10608
+10736,10737,10738,10739,10740,10741,10742,10743,10744,10745,10746,10747,10748,10749,10750,10751, #10624
+10752,10753,10754,10755,10756,10757,10758,10759,10760,10761,4946,10762,10763,10764,10765,10766, #10640
+10767,4947,4948,10768,10769,10770,10771,10772,10773,10774,10775,10776,10777,10778,10779,10780, #10656
+10781,10782,10783,10784,10785,10786,10787,10788,10789,10790,10791,10792,10793,10794,10795,10796, #10672
+10797,10798,10799,10800,10801,10802,10803,10804,10805,10806,10807,10808,10809,10810,10811,10812, #10688
+10813,10814,10815,10816,10817,10818,10819,10820,10821,10822,10823,10824,10825,10826,10827,10828, #10704
+10829,10830,10831,10832,10833,10834,10835,10836,10837,10838,10839,10840,10841,10842,10843,10844, #10720
+10845,10846,10847,10848,10849,10850,10851,10852,10853,10854,10855,10856,10857,10858,10859,10860, #10736
+10861,10862,10863,10864,10865,10866,10867,10868,10869,10870,10871,10872,10873,10874,10875,10876, #10752
+10877,10878,4486,10879,10880,10881,10882,10883,10884,10885,4949,10886,10887,10888,10889,10890, #10768
+10891,10892,10893,10894,10895,10896,10897,10898,10899,10900,10901,10902,10903,10904,10905,10906, #10784
+10907,10908,10909,10910,10911,10912,10913,10914,10915,10916,10917,10918,10919,4487,10920,10921, #10800
+10922,10923,10924,10925,10926,10927,10928,10929,10930,10931,10932,4950,10933,10934,10935,10936, #10816
+10937,10938,10939,10940,10941,10942,10943,10944,10945,10946,10947,10948,10949,4488,10950,10951, #10832
+10952,10953,10954,10955,10956,10957,10958,10959,4190,10960,10961,10962,10963,10964,10965,10966, #10848
+10967,10968,10969,10970,10971,10972,10973,10974,10975,10976,10977,10978,10979,10980,10981,10982, #10864
+10983,10984,10985,10986,10987,10988,10989,10990,10991,10992,10993,10994,10995,10996,10997,10998, #10880
+10999,11000,11001,11002,11003,11004,11005,11006,3960,11007,11008,11009,11010,11011,11012,11013, #10896
+11014,11015,11016,11017,11018,11019,11020,11021,11022,11023,11024,11025,11026,11027,11028,11029, #10912
+11030,11031,11032,4951,11033,11034,11035,11036,11037,11038,11039,11040,11041,11042,11043,11044, #10928
+11045,11046,11047,4489,11048,11049,11050,11051,4952,11052,11053,11054,11055,11056,11057,11058, #10944
+4953,11059,11060,11061,11062,11063,11064,11065,11066,11067,11068,11069,11070,11071,4954,11072, #10960
+11073,11074,11075,11076,11077,11078,11079,11080,11081,11082,11083,11084,11085,11086,11087,11088, #10976
+11089,11090,11091,11092,11093,11094,11095,11096,11097,11098,11099,11100,11101,11102,11103,11104, #10992
+11105,11106,11107,11108,11109,11110,11111,11112,11113,11114,11115,3808,11116,11117,11118,11119, #11008
+11120,11121,11122,11123,11124,11125,11126,11127,11128,11129,11130,11131,11132,11133,11134,4955, #11024
+11135,11136,11137,11138,11139,11140,11141,11142,11143,11144,11145,11146,11147,11148,11149,11150, #11040
+11151,11152,11153,11154,11155,11156,11157,11158,11159,11160,11161,4956,11162,11163,11164,11165, #11056
+11166,11167,11168,11169,11170,11171,11172,11173,11174,11175,11176,11177,11178,11179,11180,4957, #11072
+11181,11182,11183,11184,11185,11186,4958,11187,11188,11189,11190,11191,11192,11193,11194,11195, #11088
+11196,11197,11198,11199,11200,3678,11201,11202,11203,11204,11205,11206,4191,11207,11208,11209, #11104
+11210,11211,11212,11213,11214,11215,11216,11217,11218,11219,11220,11221,11222,11223,11224,11225, #11120
+11226,11227,11228,11229,11230,11231,11232,11233,11234,11235,11236,11237,11238,11239,11240,11241, #11136
+11242,11243,11244,11245,11246,11247,11248,11249,11250,11251,4959,11252,11253,11254,11255,11256, #11152
+11257,11258,11259,11260,11261,11262,11263,11264,11265,11266,11267,11268,11269,11270,11271,11272, #11168
+11273,11274,11275,11276,11277,11278,11279,11280,11281,11282,11283,11284,11285,11286,11287,11288, #11184
+11289,11290,11291,11292,11293,11294,11295,11296,11297,11298,11299,11300,11301,11302,11303,11304, #11200
+11305,11306,11307,11308,11309,11310,11311,11312,11313,11314,3679,11315,11316,11317,11318,4490, #11216
+11319,11320,11321,11322,11323,11324,11325,11326,11327,11328,11329,11330,11331,11332,11333,11334, #11232
+11335,11336,11337,11338,11339,11340,11341,11342,11343,11344,11345,11346,11347,4960,11348,11349, #11248
+11350,11351,11352,11353,11354,11355,11356,11357,11358,11359,11360,11361,11362,11363,11364,11365, #11264
+11366,11367,11368,11369,11370,11371,11372,11373,11374,11375,11376,11377,3961,4961,11378,11379, #11280
+11380,11381,11382,11383,11384,11385,11386,11387,11388,11389,11390,11391,11392,11393,11394,11395, #11296
+11396,11397,4192,11398,11399,11400,11401,11402,11403,11404,11405,11406,11407,11408,11409,11410, #11312
+11411,4962,11412,11413,11414,11415,11416,11417,11418,11419,11420,11421,11422,11423,11424,11425, #11328
+11426,11427,11428,11429,11430,11431,11432,11433,11434,11435,11436,11437,11438,11439,11440,11441, #11344
+11442,11443,11444,11445,11446,11447,11448,11449,11450,11451,11452,11453,11454,11455,11456,11457, #11360
+11458,11459,11460,11461,11462,11463,11464,11465,11466,11467,11468,11469,4963,11470,11471,4491, #11376
+11472,11473,11474,11475,4964,11476,11477,11478,11479,11480,11481,11482,11483,11484,11485,11486, #11392
+11487,11488,11489,11490,11491,11492,4965,11493,11494,11495,11496,11497,11498,11499,11500,11501, #11408
+11502,11503,11504,11505,11506,11507,11508,11509,11510,11511,11512,11513,11514,11515,11516,11517, #11424
+11518,11519,11520,11521,11522,11523,11524,11525,11526,11527,11528,11529,3962,11530,11531,11532, #11440
+11533,11534,11535,11536,11537,11538,11539,11540,11541,11542,11543,11544,11545,11546,11547,11548, #11456
+11549,11550,11551,11552,11553,11554,11555,11556,11557,11558,11559,11560,11561,11562,11563,11564, #11472
+4193,4194,11565,11566,11567,11568,11569,11570,11571,11572,11573,11574,11575,11576,11577,11578, #11488
+11579,11580,11581,11582,11583,11584,11585,11586,11587,11588,11589,11590,11591,4966,4195,11592, #11504
+11593,11594,11595,11596,11597,11598,11599,11600,11601,11602,11603,11604,3090,11605,11606,11607, #11520
+11608,11609,11610,4967,11611,11612,11613,11614,11615,11616,11617,11618,11619,11620,11621,11622, #11536
+11623,11624,11625,11626,11627,11628,11629,11630,11631,11632,11633,11634,11635,11636,11637,11638, #11552
+11639,11640,11641,11642,11643,11644,11645,11646,11647,11648,11649,11650,11651,11652,11653,11654, #11568
+11655,11656,11657,11658,11659,11660,11661,11662,11663,11664,11665,11666,11667,11668,11669,11670, #11584
+11671,11672,11673,11674,4968,11675,11676,11677,11678,11679,11680,11681,11682,11683,11684,11685, #11600
+11686,11687,11688,11689,11690,11691,11692,11693,3809,11694,11695,11696,11697,11698,11699,11700, #11616
+11701,11702,11703,11704,11705,11706,11707,11708,11709,11710,11711,11712,11713,11714,11715,11716, #11632
+11717,11718,3553,11719,11720,11721,11722,11723,11724,11725,11726,11727,11728,11729,11730,4969, #11648
+11731,11732,11733,11734,11735,11736,11737,11738,11739,11740,4492,11741,11742,11743,11744,11745, #11664
+11746,11747,11748,11749,11750,11751,11752,4970,11753,11754,11755,11756,11757,11758,11759,11760, #11680
+11761,11762,11763,11764,11765,11766,11767,11768,11769,11770,11771,11772,11773,11774,11775,11776, #11696
+11777,11778,11779,11780,11781,11782,11783,11784,11785,11786,11787,11788,11789,11790,4971,11791, #11712
+11792,11793,11794,11795,11796,11797,4972,11798,11799,11800,11801,11802,11803,11804,11805,11806, #11728
+11807,11808,11809,11810,4973,11811,11812,11813,11814,11815,11816,11817,11818,11819,11820,11821, #11744
+11822,11823,11824,11825,11826,11827,11828,11829,11830,11831,11832,11833,11834,3680,3810,11835, #11760
+11836,4974,11837,11838,11839,11840,11841,11842,11843,11844,11845,11846,11847,11848,11849,11850, #11776
+11851,11852,11853,11854,11855,11856,11857,11858,11859,11860,11861,11862,11863,11864,11865,11866, #11792
+11867,11868,11869,11870,11871,11872,11873,11874,11875,11876,11877,11878,11879,11880,11881,11882, #11808
+11883,11884,4493,11885,11886,11887,11888,11889,11890,11891,11892,11893,11894,11895,11896,11897, #11824
+11898,11899,11900,11901,11902,11903,11904,11905,11906,11907,11908,11909,11910,11911,11912,11913, #11840
+11914,11915,4975,11916,11917,11918,11919,11920,11921,11922,11923,11924,11925,11926,11927,11928, #11856
+11929,11930,11931,11932,11933,11934,11935,11936,11937,11938,11939,11940,11941,11942,11943,11944, #11872
+11945,11946,11947,11948,11949,4976,11950,11951,11952,11953,11954,11955,11956,11957,11958,11959, #11888
+11960,11961,11962,11963,11964,11965,11966,11967,11968,11969,11970,11971,11972,11973,11974,11975, #11904
+11976,11977,11978,11979,11980,11981,11982,11983,11984,11985,11986,11987,4196,11988,11989,11990, #11920
+11991,11992,4977,11993,11994,11995,11996,11997,11998,11999,12000,12001,12002,12003,12004,12005, #11936
+12006,12007,12008,12009,12010,12011,12012,12013,12014,12015,12016,12017,12018,12019,12020,12021, #11952
+12022,12023,12024,12025,12026,12027,12028,12029,12030,12031,12032,12033,12034,12035,12036,12037, #11968
+12038,12039,12040,12041,12042,12043,12044,12045,12046,12047,12048,12049,12050,12051,12052,12053, #11984
+12054,12055,12056,12057,12058,12059,12060,12061,4978,12062,12063,12064,12065,12066,12067,12068, #12000
+12069,12070,12071,12072,12073,12074,12075,12076,12077,12078,12079,12080,12081,12082,12083,12084, #12016
+12085,12086,12087,12088,12089,12090,12091,12092,12093,12094,12095,12096,12097,12098,12099,12100, #12032
+12101,12102,12103,12104,12105,12106,12107,12108,12109,12110,12111,12112,12113,12114,12115,12116, #12048
+12117,12118,12119,12120,12121,12122,12123,4979,12124,12125,12126,12127,12128,4197,12129,12130, #12064
+12131,12132,12133,12134,12135,12136,12137,12138,12139,12140,12141,12142,12143,12144,12145,12146, #12080
+12147,12148,12149,12150,12151,12152,12153,12154,4980,12155,12156,12157,12158,12159,12160,4494, #12096
+12161,12162,12163,12164,3811,12165,12166,12167,12168,12169,4495,12170,12171,4496,12172,12173, #12112
+12174,12175,12176,3812,12177,12178,12179,12180,12181,12182,12183,12184,12185,12186,12187,12188, #12128
+12189,12190,12191,12192,12193,12194,12195,12196,12197,12198,12199,12200,12201,12202,12203,12204, #12144
+12205,12206,12207,12208,12209,12210,12211,12212,12213,12214,12215,12216,12217,12218,12219,12220, #12160
+12221,4981,12222,12223,12224,12225,12226,12227,12228,12229,12230,12231,12232,12233,12234,12235, #12176
+4982,12236,12237,12238,12239,12240,12241,12242,12243,12244,12245,4983,12246,12247,12248,12249, #12192
+4984,12250,12251,12252,12253,12254,12255,12256,12257,12258,12259,12260,12261,12262,12263,12264, #12208
+4985,12265,4497,12266,12267,12268,12269,12270,12271,12272,12273,12274,12275,12276,12277,12278, #12224
+12279,12280,12281,12282,12283,12284,12285,12286,12287,4986,12288,12289,12290,12291,12292,12293, #12240
+12294,12295,12296,2473,12297,12298,12299,12300,12301,12302,12303,12304,12305,12306,12307,12308, #12256
+12309,12310,12311,12312,12313,12314,12315,12316,12317,12318,12319,3963,12320,12321,12322,12323, #12272
+12324,12325,12326,12327,12328,12329,12330,12331,12332,4987,12333,12334,12335,12336,12337,12338, #12288
+12339,12340,12341,12342,12343,12344,12345,12346,12347,12348,12349,12350,12351,12352,12353,12354, #12304
+12355,12356,12357,12358,12359,3964,12360,12361,12362,12363,12364,12365,12366,12367,12368,12369, #12320
+12370,3965,12371,12372,12373,12374,12375,12376,12377,12378,12379,12380,12381,12382,12383,12384, #12336
+12385,12386,12387,12388,12389,12390,12391,12392,12393,12394,12395,12396,12397,12398,12399,12400, #12352
+12401,12402,12403,12404,12405,12406,12407,12408,4988,12409,12410,12411,12412,12413,12414,12415, #12368
+12416,12417,12418,12419,12420,12421,12422,12423,12424,12425,12426,12427,12428,12429,12430,12431, #12384
+12432,12433,12434,12435,12436,12437,12438,3554,12439,12440,12441,12442,12443,12444,12445,12446, #12400
+12447,12448,12449,12450,12451,12452,12453,12454,12455,12456,12457,12458,12459,12460,12461,12462, #12416
+12463,12464,4989,12465,12466,12467,12468,12469,12470,12471,12472,12473,12474,12475,12476,12477, #12432
+12478,12479,12480,4990,12481,12482,12483,12484,12485,12486,12487,12488,12489,4498,12490,12491, #12448
+12492,12493,12494,12495,12496,12497,12498,12499,12500,12501,12502,12503,12504,12505,12506,12507, #12464
+12508,12509,12510,12511,12512,12513,12514,12515,12516,12517,12518,12519,12520,12521,12522,12523, #12480
+12524,12525,12526,12527,12528,12529,12530,12531,12532,12533,12534,12535,12536,12537,12538,12539, #12496
+12540,12541,12542,12543,12544,12545,12546,12547,12548,12549,12550,12551,4991,12552,12553,12554, #12512
+12555,12556,12557,12558,12559,12560,12561,12562,12563,12564,12565,12566,12567,12568,12569,12570, #12528
+12571,12572,12573,12574,12575,12576,12577,12578,3036,12579,12580,12581,12582,12583,3966,12584, #12544
+12585,12586,12587,12588,12589,12590,12591,12592,12593,12594,12595,12596,12597,12598,12599,12600, #12560
+12601,12602,12603,12604,12605,12606,12607,12608,12609,12610,12611,12612,12613,12614,12615,12616, #12576
+12617,12618,12619,12620,12621,12622,12623,12624,12625,12626,12627,12628,12629,12630,12631,12632, #12592
+12633,12634,12635,12636,12637,12638,12639,12640,12641,12642,12643,12644,12645,12646,4499,12647, #12608
+12648,12649,12650,12651,12652,12653,12654,12655,12656,12657,12658,12659,12660,12661,12662,12663, #12624
+12664,12665,12666,12667,12668,12669,12670,12671,12672,12673,12674,12675,12676,12677,12678,12679, #12640
+12680,12681,12682,12683,12684,12685,12686,12687,12688,12689,12690,12691,12692,12693,12694,12695, #12656
+12696,12697,12698,4992,12699,12700,12701,12702,12703,12704,12705,12706,12707,12708,12709,12710, #12672
+12711,12712,12713,12714,12715,12716,12717,12718,12719,12720,12721,12722,12723,12724,12725,12726, #12688
+12727,12728,12729,12730,12731,12732,12733,12734,12735,12736,12737,12738,12739,12740,12741,12742, #12704
+12743,12744,12745,12746,12747,12748,12749,12750,12751,12752,12753,12754,12755,12756,12757,12758, #12720
+12759,12760,12761,12762,12763,12764,12765,12766,12767,12768,12769,12770,12771,12772,12773,12774, #12736
+12775,12776,12777,12778,4993,2175,12779,12780,12781,12782,12783,12784,12785,12786,4500,12787, #12752
+12788,12789,12790,12791,12792,12793,12794,12795,12796,12797,12798,12799,12800,12801,12802,12803, #12768
+12804,12805,12806,12807,12808,12809,12810,12811,12812,12813,12814,12815,12816,12817,12818,12819, #12784
+12820,12821,12822,12823,12824,12825,12826,4198,3967,12827,12828,12829,12830,12831,12832,12833, #12800
+12834,12835,12836,12837,12838,12839,12840,12841,12842,12843,12844,12845,12846,12847,12848,12849, #12816
+12850,12851,12852,12853,12854,12855,12856,12857,12858,12859,12860,12861,4199,12862,12863,12864, #12832
+12865,12866,12867,12868,12869,12870,12871,12872,12873,12874,12875,12876,12877,12878,12879,12880, #12848
+12881,12882,12883,12884,12885,12886,12887,4501,12888,12889,12890,12891,12892,12893,12894,12895, #12864
+12896,12897,12898,12899,12900,12901,12902,12903,12904,12905,12906,12907,12908,12909,12910,12911, #12880
+12912,4994,12913,12914,12915,12916,12917,12918,12919,12920,12921,12922,12923,12924,12925,12926, #12896
+12927,12928,12929,12930,12931,12932,12933,12934,12935,12936,12937,12938,12939,12940,12941,12942, #12912
+12943,12944,12945,12946,12947,12948,12949,12950,12951,12952,12953,12954,12955,12956,1772,12957, #12928
+12958,12959,12960,12961,12962,12963,12964,12965,12966,12967,12968,12969,12970,12971,12972,12973, #12944
+12974,12975,12976,12977,12978,12979,12980,12981,12982,12983,12984,12985,12986,12987,12988,12989, #12960
+12990,12991,12992,12993,12994,12995,12996,12997,4502,12998,4503,12999,13000,13001,13002,13003, #12976
+4504,13004,13005,13006,13007,13008,13009,13010,13011,13012,13013,13014,13015,13016,13017,13018, #12992
+13019,13020,13021,13022,13023,13024,13025,13026,13027,13028,13029,3449,13030,13031,13032,13033, #13008
+13034,13035,13036,13037,13038,13039,13040,13041,13042,13043,13044,13045,13046,13047,13048,13049, #13024
+13050,13051,13052,13053,13054,13055,13056,13057,13058,13059,13060,13061,13062,13063,13064,13065, #13040
+13066,13067,13068,13069,13070,13071,13072,13073,13074,13075,13076,13077,13078,13079,13080,13081, #13056
+13082,13083,13084,13085,13086,13087,13088,13089,13090,13091,13092,13093,13094,13095,13096,13097, #13072
+13098,13099,13100,13101,13102,13103,13104,13105,13106,13107,13108,13109,13110,13111,13112,13113, #13088
+13114,13115,13116,13117,13118,3968,13119,4995,13120,13121,13122,13123,13124,13125,13126,13127, #13104
+4505,13128,13129,13130,13131,13132,13133,13134,4996,4506,13135,13136,13137,13138,13139,4997, #13120
+13140,13141,13142,13143,13144,13145,13146,13147,13148,13149,13150,13151,13152,13153,13154,13155, #13136
+13156,13157,13158,13159,4998,13160,13161,13162,13163,13164,13165,13166,13167,13168,13169,13170, #13152
+13171,13172,13173,13174,13175,13176,4999,13177,13178,13179,13180,13181,13182,13183,13184,13185, #13168
+13186,13187,13188,13189,13190,13191,13192,13193,13194,13195,13196,13197,13198,13199,13200,13201, #13184
+13202,13203,13204,13205,13206,5000,13207,13208,13209,13210,13211,13212,13213,13214,13215,13216, #13200
+13217,13218,13219,13220,13221,13222,13223,13224,13225,13226,13227,4200,5001,13228,13229,13230, #13216
+13231,13232,13233,13234,13235,13236,13237,13238,13239,13240,3969,13241,13242,13243,13244,3970, #13232
+13245,13246,13247,13248,13249,13250,13251,13252,13253,13254,13255,13256,13257,13258,13259,13260, #13248
+13261,13262,13263,13264,13265,13266,13267,13268,3450,13269,13270,13271,13272,13273,13274,13275, #13264
+13276,5002,13277,13278,13279,13280,13281,13282,13283,13284,13285,13286,13287,13288,13289,13290, #13280
+13291,13292,13293,13294,13295,13296,13297,13298,13299,13300,13301,13302,3813,13303,13304,13305, #13296
+13306,13307,13308,13309,13310,13311,13312,13313,13314,13315,13316,13317,13318,13319,13320,13321, #13312
+13322,13323,13324,13325,13326,13327,13328,4507,13329,13330,13331,13332,13333,13334,13335,13336, #13328
+13337,13338,13339,13340,13341,5003,13342,13343,13344,13345,13346,13347,13348,13349,13350,13351, #13344
+13352,13353,13354,13355,13356,13357,13358,13359,13360,13361,13362,13363,13364,13365,13366,13367, #13360
+5004,13368,13369,13370,13371,13372,13373,13374,13375,13376,13377,13378,13379,13380,13381,13382, #13376
+13383,13384,13385,13386,13387,13388,13389,13390,13391,13392,13393,13394,13395,13396,13397,13398, #13392
+13399,13400,13401,13402,13403,13404,13405,13406,13407,13408,13409,13410,13411,13412,13413,13414, #13408
+13415,13416,13417,13418,13419,13420,13421,13422,13423,13424,13425,13426,13427,13428,13429,13430, #13424
+13431,13432,4508,13433,13434,13435,4201,13436,13437,13438,13439,13440,13441,13442,13443,13444, #13440
+13445,13446,13447,13448,13449,13450,13451,13452,13453,13454,13455,13456,13457,5005,13458,13459, #13456
+13460,13461,13462,13463,13464,13465,13466,13467,13468,13469,13470,4509,13471,13472,13473,13474, #13472
+13475,13476,13477,13478,13479,13480,13481,13482,13483,13484,13485,13486,13487,13488,13489,13490, #13488
+13491,13492,13493,13494,13495,13496,13497,13498,13499,13500,13501,13502,13503,13504,13505,13506, #13504
+13507,13508,13509,13510,13511,13512,13513,13514,13515,13516,13517,13518,13519,13520,13521,13522, #13520
+13523,13524,13525,13526,13527,13528,13529,13530,13531,13532,13533,13534,13535,13536,13537,13538, #13536
+13539,13540,13541,13542,13543,13544,13545,13546,13547,13548,13549,13550,13551,13552,13553,13554, #13552
+13555,13556,13557,13558,13559,13560,13561,13562,13563,13564,13565,13566,13567,13568,13569,13570, #13568
+13571,13572,13573,13574,13575,13576,13577,13578,13579,13580,13581,13582,13583,13584,13585,13586, #13584
+13587,13588,13589,13590,13591,13592,13593,13594,13595,13596,13597,13598,13599,13600,13601,13602, #13600
+13603,13604,13605,13606,13607,13608,13609,13610,13611,13612,13613,13614,13615,13616,13617,13618, #13616
+13619,13620,13621,13622,13623,13624,13625,13626,13627,13628,13629,13630,13631,13632,13633,13634, #13632
+13635,13636,13637,13638,13639,13640,13641,13642,5006,13643,13644,13645,13646,13647,13648,13649, #13648
+13650,13651,5007,13652,13653,13654,13655,13656,13657,13658,13659,13660,13661,13662,13663,13664, #13664
+13665,13666,13667,13668,13669,13670,13671,13672,13673,13674,13675,13676,13677,13678,13679,13680, #13680
+13681,13682,13683,13684,13685,13686,13687,13688,13689,13690,13691,13692,13693,13694,13695,13696, #13696
+13697,13698,13699,13700,13701,13702,13703,13704,13705,13706,13707,13708,13709,13710,13711,13712, #13712
+13713,13714,13715,13716,13717,13718,13719,13720,13721,13722,13723,13724,13725,13726,13727,13728, #13728
+13729,13730,13731,13732,13733,13734,13735,13736,13737,13738,13739,13740,13741,13742,13743,13744, #13744
+13745,13746,13747,13748,13749,13750,13751,13752,13753,13754,13755,13756,13757,13758,13759,13760, #13760
+13761,13762,13763,13764,13765,13766,13767,13768,13769,13770,13771,13772,13773,13774,3273,13775, #13776
+13776,13777,13778,13779,13780,13781,13782,13783,13784,13785,13786,13787,13788,13789,13790,13791, #13792
+13792,13793,13794,13795,13796,13797,13798,13799,13800,13801,13802,13803,13804,13805,13806,13807, #13808
+13808,13809,13810,13811,13812,13813,13814,13815,13816,13817,13818,13819,13820,13821,13822,13823, #13824
+13824,13825,13826,13827,13828,13829,13830,13831,13832,13833,13834,13835,13836,13837,13838,13839, #13840
+13840,13841,13842,13843,13844,13845,13846,13847,13848,13849,13850,13851,13852,13853,13854,13855, #13856
+13856,13857,13858,13859,13860,13861,13862,13863,13864,13865,13866,13867,13868,13869,13870,13871, #13872
+13872,13873,13874,13875,13876,13877,13878,13879,13880,13881,13882,13883,13884,13885,13886,13887, #13888
+13888,13889,13890,13891,13892,13893,13894,13895,13896,13897,13898,13899,13900,13901,13902,13903, #13904
+13904,13905,13906,13907,13908,13909,13910,13911,13912,13913,13914,13915,13916,13917,13918,13919, #13920
+13920,13921,13922,13923,13924,13925,13926,13927,13928,13929,13930,13931,13932,13933,13934,13935, #13936
+13936,13937,13938,13939,13940,13941,13942,13943,13944,13945,13946,13947,13948,13949,13950,13951, #13952
+13952,13953,13954,13955,13956,13957,13958,13959,13960,13961,13962,13963,13964,13965,13966,13967, #13968
+13968,13969,13970,13971,13972) #13973

libs/requests/packages/chardet/big5prober.py

@@ -0,0 +1,41 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is Mozilla Communicator client code.
+# 
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 1998
+# the Initial Developer. All Rights Reserved.
+# 
+# Contributor(s):
+#   Mark Pilgrim - port to Python
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+# 
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+# 
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+from mbcharsetprober import MultiByteCharSetProber
+from codingstatemachine import CodingStateMachine
+from chardistribution import Big5DistributionAnalysis
+from mbcssm import Big5SMModel
+
+class Big5Prober(MultiByteCharSetProber):
+    def __init__(self):
+        MultiByteCharSetProber.__init__(self)
+        self._mCodingSM = CodingStateMachine(Big5SMModel)
+        self._mDistributionAnalyzer = Big5DistributionAnalysis()
+        self.reset()
+
+    def get_charset_name(self):
+        return "Big5"

libs/requests/packages/chardet/chardistribution.py

@@ -0,0 +1,200 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is Mozilla Communicator client code.
+# 
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 1998
+# the Initial Developer. All Rights Reserved.
+# 
+# Contributor(s):
+#   Mark Pilgrim - port to Python
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+# 
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+# 
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+import constants
+from euctwfreq import EUCTWCharToFreqOrder, EUCTW_TABLE_SIZE, EUCTW_TYPICAL_DISTRIBUTION_RATIO
+from euckrfreq import EUCKRCharToFreqOrder, EUCKR_TABLE_SIZE, EUCKR_TYPICAL_DISTRIBUTION_RATIO
+from gb2312freq import GB2312CharToFreqOrder, GB2312_TABLE_SIZE, GB2312_TYPICAL_DISTRIBUTION_RATIO
+from big5freq import Big5CharToFreqOrder, BIG5_TABLE_SIZE, BIG5_TYPICAL_DISTRIBUTION_RATIO
+from jisfreq import JISCharToFreqOrder, JIS_TABLE_SIZE, JIS_TYPICAL_DISTRIBUTION_RATIO
+
+ENOUGH_DATA_THRESHOLD = 1024
+SURE_YES = 0.99
+SURE_NO = 0.01
+
+class CharDistributionAnalysis:
+    def __init__(self):
+        self._mCharToFreqOrder = None # Mapping table to get frequency order from char order (get from GetOrder())
+        self._mTableSize = None # Size of above table
+        self._mTypicalDistributionRatio = None # This is a constant value which varies from language to language, used in calculating confidence.  See http://www.mozilla.org/projects/intl/UniversalCharsetDetection.html for further detail.
+        self.reset()
+        
+    def reset(self):
+        """reset analyser, clear any state"""
+        self._mDone = constants.False # If this flag is set to constants.True, detection is done and conclusion has been made
+        self._mTotalChars = 0 # Total characters encountered
+        self._mFreqChars = 0 # The number of characters whose frequency order is less than 512
+
+    def feed(self, aStr, aCharLen):
+        """feed a character with known length"""
+        if aCharLen == 2:
+            # we only care about 2-bytes character in our distribution analysis
+            order = self.get_order(aStr)
+        else:
+            order = -1
+        if order >= 0:
+            self._mTotalChars += 1
+            # order is valid
+            if order < self._mTableSize:
+                if 512 > self._mCharToFreqOrder[order]:
+                    self._mFreqChars += 1
+
+    def get_confidence(self):
+        """return confidence based on existing data"""
+        # if we didn't receive any character in our consideration range, return negative answer
+        if self._mTotalChars <= 0:
+            return SURE_NO
+
+        if self._mTotalChars != self._mFreqChars:
+            r = self._mFreqChars / ((self._mTotalChars - self._mFreqChars) * self._mTypicalDistributionRatio)
+            if r < SURE_YES:
+                return r
+
+        # normalize confidence (we don't want to be 100% sure)
+        return SURE_YES
+
+    def got_enough_data(self):
+        # It is not necessary to receive all data to draw conclusion. For charset detection,
+        # certain amount of data is enough
+        return self._mTotalChars > ENOUGH_DATA_THRESHOLD
+
+    def get_order(self, aStr):
+        # We do not handle characters based on the original encoding string, but 
+        # convert this encoding string to a number, here called order.
+        # This allows multiple encodings of a language to share one frequency table.
+        return -1
+    
+class EUCTWDistributionAnalysis(CharDistributionAnalysis):
+    def __init__(self):
+        CharDistributionAnalysis.__init__(self)
+        self._mCharToFreqOrder = EUCTWCharToFreqOrder
+        self._mTableSize = EUCTW_TABLE_SIZE
+        self._mTypicalDistributionRatio = EUCTW_TYPICAL_DISTRIBUTION_RATIO
+
+    def get_order(self, aStr):
+        # for euc-TW encoding, we are interested 
+        #   first  byte range: 0xc4 -- 0xfe
+        #   second byte range: 0xa1 -- 0xfe
+        # no validation needed here. State machine has done that
+        if aStr[0] >= '\xC4':
+            return 94 * (ord(aStr[0]) - 0xC4) + ord(aStr[1]) - 0xA1
+        else:
+            return -1
+
+class EUCKRDistributionAnalysis(CharDistributionAnalysis):
+    def __init__(self):
+        CharDistributionAnalysis.__init__(self)
+        self._mCharToFreqOrder = EUCKRCharToFreqOrder
+        self._mTableSize = EUCKR_TABLE_SIZE
+        self._mTypicalDistributionRatio = EUCKR_TYPICAL_DISTRIBUTION_RATIO
+
+    def get_order(self, aStr):
+        # for euc-KR encoding, we are interested 
+        #   first  byte range: 0xb0 -- 0xfe
+        #   second byte range: 0xa1 -- 0xfe
+        # no validation needed here. State machine has done that
+        if aStr[0] >= '\xB0':
+            return 94 * (ord(aStr[0]) - 0xB0) + ord(aStr[1]) - 0xA1
+        else:
+            return -1;
+
+class GB2312DistributionAnalysis(CharDistributionAnalysis):
+    def __init__(self):
+        CharDistributionAnalysis.__init__(self)
+        self._mCharToFreqOrder = GB2312CharToFreqOrder
+        self._mTableSize = GB2312_TABLE_SIZE
+        self._mTypicalDistributionRatio = GB2312_TYPICAL_DISTRIBUTION_RATIO
+
+    def get_order(self, aStr):
+        # for GB2312 encoding, we are interested 
+        #  first  byte range: 0xb0 -- 0xfe
+        #  second byte range: 0xa1 -- 0xfe
+        # no validation needed here. State machine has done that
+        if (aStr[0] >= '\xB0') and (aStr[1] >= '\xA1'):
+            return 94 * (ord(aStr[0]) - 0xB0) + ord(aStr[1]) - 0xA1
+        else:
+            return -1;
+
+class Big5DistributionAnalysis(CharDistributionAnalysis):
+    def __init__(self):
+        CharDistributionAnalysis.__init__(self)
+        self._mCharToFreqOrder = Big5CharToFreqOrder
+        self._mTableSize = BIG5_TABLE_SIZE
+        self._mTypicalDistributionRatio = BIG5_TYPICAL_DISTRIBUTION_RATIO
+
+    def get_order(self, aStr):
+        # for big5 encoding, we are interested 
+        #   first  byte range: 0xa4 -- 0xfe
+        #   second byte range: 0x40 -- 0x7e , 0xa1 -- 0xfe
+        # no validation needed here. State machine has done that
+        if aStr[0] >= '\xA4':
+            if aStr[1] >= '\xA1':
+                return 157 * (ord(aStr[0]) - 0xA4) + ord(aStr[1]) - 0xA1 + 63
+            else:
+                return 157 * (ord(aStr[0]) - 0xA4) + ord(aStr[1]) - 0x40
+        else:
+            return -1
+
+class SJISDistributionAnalysis(CharDistributionAnalysis):
+    def __init__(self):
+        CharDistributionAnalysis.__init__(self)
+        self._mCharToFreqOrder = JISCharToFreqOrder
+        self._mTableSize = JIS_TABLE_SIZE
+        self._mTypicalDistributionRatio = JIS_TYPICAL_DISTRIBUTION_RATIO
+
+    def get_order(self, aStr):
+        # for sjis encoding, we are interested 
+        #   first  byte range: 0x81 -- 0x9f , 0xe0 -- 0xfe
+        #   second byte range: 0x40 -- 0x7e,  0x81 -- oxfe
+        # no validation needed here. State machine has done that
+        if (aStr[0] >= '\x81') and (aStr[0] <= '\x9F'):
+            order = 188 * (ord(aStr[0]) - 0x81)
+        elif (aStr[0] >= '\xE0') and (aStr[0] <= '\xEF'):
+            order = 188 * (ord(aStr[0]) - 0xE0 + 31)
+        else:
+            return -1;
+        order = order + ord(aStr[1]) - 0x40
+        if aStr[1] > '\x7F':
+            order =- 1
+        return order
+
+class EUCJPDistributionAnalysis(CharDistributionAnalysis):
+    def __init__(self):
+        CharDistributionAnalysis.__init__(self)
+        self._mCharToFreqOrder = JISCharToFreqOrder
+        self._mTableSize = JIS_TABLE_SIZE
+        self._mTypicalDistributionRatio = JIS_TYPICAL_DISTRIBUTION_RATIO
+
+    def get_order(self, aStr):
+        # for euc-JP encoding, we are interested 
+        #   first  byte range: 0xa0 -- 0xfe
+        #   second byte range: 0xa1 -- 0xfe
+        # no validation needed here. State machine has done that
+        if aStr[0] >= '\xA0':
+            return 94 * (ord(aStr[0]) - 0xA1) + ord(aStr[1]) - 0xa1
+        else:
+            return -1

libs/requests/packages/chardet/charsetgroupprober.py

@@ -0,0 +1,96 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is Mozilla Communicator client code.
+# 
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 1998
+# the Initial Developer. All Rights Reserved.
+# 
+# Contributor(s):
+#   Mark Pilgrim - port to Python
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+# 
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+# 
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+import constants, sys
+from charsetprober import CharSetProber
+
+class CharSetGroupProber(CharSetProber):
+    def __init__(self):
+        CharSetProber.__init__(self)
+        self._mActiveNum = 0
+        self._mProbers = []
+        self._mBestGuessProber = None
+        
+    def reset(self):
+        CharSetProber.reset(self)
+        self._mActiveNum = 0
+        for prober in self._mProbers:
+            if prober:
+                prober.reset()
+                prober.active = constants.True
+                self._mActiveNum += 1
+        self._mBestGuessProber = None
+
+    def get_charset_name(self):
+        if not self._mBestGuessProber:
+            self.get_confidence()
+            if not self._mBestGuessProber: return None
+#                self._mBestGuessProber = self._mProbers[0]
+        return self._mBestGuessProber.get_charset_name()
+
+    def feed(self, aBuf):
+        for prober in self._mProbers:
+            if not prober: continue
+            if not prober.active: continue
+            st = prober.feed(aBuf)
+            if not st: continue
+            if st == constants.eFoundIt:
+                self._mBestGuessProber = prober
+                return self.get_state()
+            elif st == constants.eNotMe:
+                prober.active = constants.False
+                self._mActiveNum -= 1
+                if self._mActiveNum <= 0:
+                    self._mState = constants.eNotMe
+                    return self.get_state()
+        return self.get_state()
+
+    def get_confidence(self):
+        st = self.get_state()
+        if st == constants.eFoundIt:
+            return 0.99
+        elif st == constants.eNotMe:
+            return 0.01
+        bestConf = 0.0
+        self._mBestGuessProber = None
+        for prober in self._mProbers:
+            if not prober: continue
+            if not prober.active:
+                if constants._debug:
+                    sys.stderr.write(prober.get_charset_name() + ' not active\n')
+                continue
+            cf = prober.get_confidence()
+            if constants._debug:
+                sys.stderr.write('%s confidence = %s\n' % (prober.get_charset_name(), cf))
+            if bestConf < cf:
+                bestConf = cf
+                self._mBestGuessProber = prober
+        if not self._mBestGuessProber: return 0.0
+        return bestConf
+#        else:
+#            self._mBestGuessProber = self._mProbers[0]
+#            return self._mBestGuessProber.get_confidence()

libs/requests/packages/chardet/charsetprober.py

@@ -0,0 +1,60 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is Mozilla Universal charset detector code.
+# 
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 2001
+# the Initial Developer. All Rights Reserved.
+# 
+# Contributor(s):
+#   Mark Pilgrim - port to Python
+#   Shy Shalom - original C code
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+# 
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+# 
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+import constants, re
+
+class CharSetProber:
+    def __init__(self):
+        pass
+        
+    def reset(self):
+        self._mState = constants.eDetecting
+    
+    def get_charset_name(self):
+        return None
+
+    def feed(self, aBuf):
+        pass
+
+    def get_state(self):
+        return self._mState
+
+    def get_confidence(self):
+        return 0.0
+
+    def filter_high_bit_only(self, aBuf):
+        aBuf = re.sub(r'([\x00-\x7F])+', ' ', aBuf)
+        return aBuf
+    
+    def filter_without_english_letters(self, aBuf):
+        aBuf = re.sub(r'([A-Za-z])+', ' ', aBuf)
+        return aBuf
+        
+    def filter_with_english_letters(self, aBuf):
+        # TODO
+        return aBuf