Merge branches 'develop' and 'manual_scan' of https://github.com/RuudBurger/CouchPotatoServer into manual_scan

Conflicts:
	couchpotato/core/plugins/renamer/main.py

contributing.md

@@ -1,15 +1,25 @@
-#So you feel like posting a bug, sending me a pull request or just telling me how awesome I am. No problem! 
+## Got a issue/feature request or submitting a pull request?
 
-##Just make sure you think of the following things:
+Make sure you think of the following things:
 
- * Search through the existing (and closed) issues first. See if you can get your answer there.
+## Issue
+ * Search through the existing (and closed) issues first, see if you can get your answer there.
  * Double check the result manually, because it could be an external issue. 
  * Post logs! Without seeing what is going on, I can't reproduce the error.
- * What is the movie + quality you are searching for.
- * What are you settings for the specific problem.
- * What providers are you using. (While your logs include these, scanning through hundred of lines of log isn't my hobby).
- * Give me a short step by step of how to reproduce.
+ * Also check the logs before submitting, obvious errors like permission or http errors are often not related to CP.
+ * What is the movie + quality you are searching for?
+ * What are you're settings for the specific problem?
+ * What providers are you using? (While you're logs include these, scanning through hundred of lines of log isn't our hobby)
+ * Post the logs from config directory, please do not copy paste the UI. Use pastebin to store these logs!
+ * Give a short step by step of how to reproduce the error.
  * What hardware / OS are you using and what are the limits? NAS can be slow and maybe have a different python installed then when you use CP on OSX or Windows for example.
- * I will mark issues with the "can't reproduce" tag. Don't go asking me "why closed" if it clearly says the issue in the tag ;)
+ * I will mark issues with the "can't reproduce" tag. Don't go asking "why closed" if it clearly says the issue in the tag ;)
+ * If you're running on a NAS (QNAP, Austor etc..) with pre-made packages, make sure these are setup to use our source repo (RuudBurger/CouchPotatoServer) and nothing else!!
 
-**If I don't get enough info, the chance of the issue getting closed is a lot bigger ;)**
+## Pull Request
+ * Make sure you're pull request is made for develop branch (or relevant feature branch)
+ * Have you tested your PR? If not, why?
+ * Are there any limitations of your PR we should know of?
+ * Make sure to keep you're PR up-to-date with the branch you're trying to push into.
+
+**If we don't get enough info, the chance of the issue getting closed is a lot bigger ;)**

couchpotato/core/_base/clientscript/main.py

@@ -34,6 +34,8 @@ class ClientScript(Plugin):
             'scripts/library/question.js',
             'scripts/library/scrollspy.js',
             'scripts/library/spin.js',
+            'scripts/library/Array.stableSort.js',
+            'scripts/library/async.js',
             'scripts/couchpotato.js',
             'scripts/api.js',
             'scripts/library/history.js',

couchpotato/core/_base/scheduler/main.py

@@ -31,13 +31,13 @@ class Scheduler(Plugin):
                 pass
 
     def doShutdown(self):
-        super(Scheduler, self).doShutdown()
         self.stop()
+        return super(Scheduler, self).doShutdown()
 
     def stop(self):
         if self.started:
             log.debug('Stopping scheduler')
-            self.sched.shutdown()
+            self.sched.shutdown(wait = False)
             log.debug('Scheduler stopped')
         self.started = False
 

couchpotato/core/_base/updater/main.py

@@ -183,9 +183,6 @@ class GitUpdater(BaseUpdater):
     def doUpdate(self):
 
         try:
-            log.debug('Stashing local changes')
-            self.repo.saveStash()
-
             log.info('Updating to latest version')
             self.repo.pull()
 

couchpotato/core/_base/updater/static/updater.js

@@ -24,7 +24,7 @@ var UpdaterBase = new Class({
 					self.doUpdate();
 				else {
 					App.unBlockPage();
-					App.fireEvent('message', 'No updates available');
+					App.on('message', 'No updates available');
 				}
 			}
 		})

couchpotato/core/downloaders/base.py

@@ -49,21 +49,26 @@ class Downloader(Provider):
 
         return []
 
-    def _download(self, data = None, movie = None, manual = False, filedata = None):
-        if not movie: movie = {}
+    def _download(self, data = None, media = None, manual = False, filedata = None):
+        if not media: media = {}
         if not data: data = {}
 
         if self.isDisabled(manual, data):
             return
-        return self.download(data = data, movie = movie, filedata = filedata)
+        return self.download(data = data, media = media, filedata = filedata)
 
-    def _getAllDownloadStatus(self):
+    def _getAllDownloadStatus(self, download_ids):
         if self.isDisabled(manual = True, data = {}):
             return
 
-        return self.getAllDownloadStatus()
+        ids = [download_id['id'] for download_id in download_ids if download_id['downloader'] == self.getName()]
 
-    def getAllDownloadStatus(self):
+        if ids:
+            return self.getAllDownloadStatus(ids)
+        else:
+            return
+
+    def getAllDownloadStatus(self, ids):
         return
 
     def _removeFailed(self, release_download):

couchpotato/core/downloaders/blackhole/__init__.py

@@ -13,7 +13,7 @@ config = [{
             'list': 'download_providers',
             'name': 'blackhole',
             'label': 'Black hole',
-            'description': 'Download the NZB/Torrent to a specific folder.',
+            'description': 'Download the NZB/Torrent to a specific folder. <em>Note: Seeding and copying/linking features do <strong>not</strong> work with Black hole</em>.',
             'wizard': True,
             'options': [
                 {

couchpotato/core/downloaders/blackhole/main.py

@@ -12,8 +12,8 @@ class Blackhole(Downloader):
 
     protocol = ['nzb', 'torrent', 'torrent_magnet']
 
-    def download(self, data = None, movie = None, filedata = None):
-        if not movie: movie = {}
+    def download(self, data = None, media = None, filedata = None):
+        if not media: media = {}
         if not data: data = {}
 
         directory = self.conf('directory')
@@ -33,7 +33,7 @@ class Blackhole(Downloader):
                         log.error('No nzb/torrent available: %s', data.get('url'))
                         return False
 
-                file_name = self.createFileName(data, filedata, movie)
+                file_name = self.createFileName(data, filedata, media)
                 full_path = os.path.join(directory, file_name)
 
                 if self.conf('create_subdir'):
@@ -51,10 +51,10 @@ class Blackhole(Downloader):
                         with open(full_path, 'wb') as f:
                             f.write(filedata)
                         os.chmod(full_path, Env.getPermission('file'))
-                        return True
+                        return self.downloadReturnId('')
                     else:
                         log.info('File %s already exists.', full_path)
-                        return True
+                        return self.downloadReturnId('')
 
                 except:
                     log.error('Failed to download to blackhole %s', traceback.format_exc())

couchpotato/core/downloaders/deluge/main.py

@@ -32,7 +32,10 @@ class Deluge(Downloader):
 
         return self.drpc
 
-    def download(self, data, movie, filedata = None):
+    def download(self, data = None, media = None, filedata = None):
+        if not media: media = {}
+        if not data: data = {}
+
         log.info('Sending "%s" (%s) to Deluge.', (data.get('name'), data.get('protocol')))
 
         if not self.connect():
@@ -73,7 +76,7 @@ class Deluge(Downloader):
         if data.get('protocol') == 'torrent_magnet':
             remote_torrent = self.drpc.add_torrent_magnet(data.get('url'), options)
         else:
-            filename = self.createFileName(data, filedata, movie)
+            filename = self.createFileName(data, filedata, media)
             remote_torrent = self.drpc.add_torrent_file(filename, filedata, options)
 
         if not remote_torrent:
@@ -83,7 +86,7 @@ class Deluge(Downloader):
         log.info('Torrent sent to Deluge successfully.')
         return self.downloadReturnId(remote_torrent)
 
-    def getAllDownloadStatus(self):
+    def getAllDownloadStatus(self, ids):
 
         log.debug('Checking Deluge download status.')
 
@@ -100,38 +103,39 @@ class Deluge(Downloader):
 
         for torrent_id in queue:
             torrent = queue[torrent_id]
-            log.debug('name=%s / id=%s / save_path=%s / move_completed_path=%s / hash=%s / progress=%s / state=%s / eta=%s / ratio=%s / stop_ratio=%s / is_seed=%s / is_finished=%s / paused=%s', (torrent['name'], torrent['hash'], torrent['save_path'], torrent['move_completed_path'], torrent['hash'], torrent['progress'], torrent['state'], torrent['eta'], torrent['ratio'], torrent['stop_ratio'], torrent['is_seed'], torrent['is_finished'], torrent['paused']))
-
-            # Deluge has no easy way to work out if a torrent is stalled or failing.
-            #status = 'failed'
-            status = 'busy'
-            if torrent['is_seed'] and tryFloat(torrent['ratio']) < tryFloat(torrent['stop_ratio']):
-                # We have torrent['seeding_time'] to work out what the seeding time is, but we do not
-                # have access to the downloader seed_time, as with deluge we have no way to pass it
-                # when the torrent is added. So Deluge will only look at the ratio.
-                # See above comment in download().
-                status = 'seeding'
-            elif torrent['is_seed'] and torrent['is_finished'] and torrent['paused'] and torrent['state'] == 'Paused':
-                status = 'completed'
-
-            download_dir = sp(torrent['save_path'])
-            if torrent['move_on_completed']:
-                download_dir = torrent['move_completed_path']
-
-            torrent_files = []
-            for file_item in torrent['files']:
-                torrent_files.append(os.path.join(download_dir), sp(file_item['path']))
-
-            release_downloads.append({
-                'id': torrent['hash'],
-                'name': torrent['name'],
-                'status': status,
-                'original_status': torrent['state'],
-                'seed_ratio': torrent['ratio'],
-                'timeleft': str(timedelta(seconds = torrent['eta'])),
-                'folder': sp(download_dir if len(torrent_files) == 1 else os.path.join(download_dir, torrent['name'])),
-                'files': '|'.join(torrent_files),
-            })
+            if torrent['hash'] in ids:
+                log.debug('name=%s / id=%s / save_path=%s / move_completed_path=%s / hash=%s / progress=%s / state=%s / eta=%s / ratio=%s / stop_ratio=%s / is_seed=%s / is_finished=%s / paused=%s', (torrent['name'], torrent['hash'], torrent['save_path'], torrent['move_completed_path'], torrent['hash'], torrent['progress'], torrent['state'], torrent['eta'], torrent['ratio'], torrent['stop_ratio'], torrent['is_seed'], torrent['is_finished'], torrent['paused']))
+    
+                # Deluge has no easy way to work out if a torrent is stalled or failing.
+                #status = 'failed'
+                status = 'busy'
+                if torrent['is_seed'] and tryFloat(torrent['ratio']) < tryFloat(torrent['stop_ratio']):
+                    # We have torrent['seeding_time'] to work out what the seeding time is, but we do not
+                    # have access to the downloader seed_time, as with deluge we have no way to pass it
+                    # when the torrent is added. So Deluge will only look at the ratio.
+                    # See above comment in download().
+                    status = 'seeding'
+                elif torrent['is_seed'] and torrent['is_finished'] and torrent['paused'] and torrent['state'] == 'Paused':
+                    status = 'completed'
+    
+                download_dir = sp(torrent['save_path'])
+                if torrent['move_on_completed']:
+                    download_dir = torrent['move_completed_path']
+    
+                torrent_files = []
+                for file_item in torrent['files']:
+                    torrent_files.append(sp(os.path.join(download_dir, file_item['path'])))
+    
+                release_downloads.append({
+                    'id': torrent['hash'],
+                    'name': torrent['name'],
+                    'status': status,
+                    'original_status': torrent['state'],
+                    'seed_ratio': torrent['ratio'],
+                    'timeleft': str(timedelta(seconds = torrent['eta'])),
+                    'folder': sp(download_dir if len(torrent_files) == 1 else os.path.join(download_dir, torrent['name'])),
+                    'files': '|'.join(torrent_files),
+                })
 
         return release_downloads
 

couchpotato/core/downloaders/nzbget/main.py

@@ -19,8 +19,8 @@ class NZBGet(Downloader):
 
     url = 'http://%(username)s:%(password)s@%(host)s/xmlrpc'
 
-    def download(self, data = None, movie = None, filedata = None):
-        if not movie: movie = {}
+    def download(self, data = None, media = None, filedata = None):
+        if not media: media = {}
         if not data: data = {}
 
         if not filedata:
@@ -30,7 +30,7 @@ class NZBGet(Downloader):
         log.info('Sending "%s" to NZBGet.', data.get('name'))
 
         url = self.url % {'host': self.conf('host'), 'username': self.conf('username'), 'password': self.conf('password')}
-        nzb_name = ss('%s.nzb' % self.createNzbName(data, movie))
+        nzb_name = ss('%s.nzb' % self.createNzbName(data, media))
 
         rpc = xmlrpclib.ServerProxy(url)
         try:
@@ -67,7 +67,7 @@ class NZBGet(Downloader):
             log.error('NZBGet could not add %s to the queue.', nzb_name)
             return False
 
-    def getAllDownloadStatus(self):
+    def getAllDownloadStatus(self, ids):
 
         log.debug('Checking NZBGet download status.')
 
@@ -102,51 +102,54 @@ class NZBGet(Downloader):
         release_downloads = ReleaseDownloadList(self)
 
         for nzb in groups:
-            log.debug('Found %s in NZBGet download queue', nzb['NZBFilename'])
             try:
                 nzb_id = [param['Value'] for param in nzb['Parameters'] if param['Name'] == 'couchpotato'][0]
             except:
                 nzb_id = nzb['NZBID']
 
-
-            timeleft = -1
-            try:
-                if nzb['ActiveDownloads'] > 0 and nzb['DownloadRate'] > 0 and not (status['DownloadPaused'] or status['Download2Paused']):
-                    timeleft = str(timedelta(seconds = nzb['RemainingSizeMB'] / status['DownloadRate'] * 2 ^ 20))
-            except:
-                pass
-
-            release_downloads.append({
-                'id': nzb_id,
-                'name': nzb['NZBFilename'],
-                'original_status': 'DOWNLOADING' if nzb['ActiveDownloads'] > 0 else 'QUEUED',
-                # Seems to have no native API function for time left. This will return the time left after NZBGet started downloading this item
-                'timeleft': timeleft,
-            })
+            if nzb_id in ids:
+                log.debug('Found %s in NZBGet download queue', nzb['NZBFilename'])
+                timeleft = -1
+                try:
+                    if nzb['ActiveDownloads'] > 0 and nzb['DownloadRate'] > 0 and not (status['DownloadPaused'] or status['Download2Paused']):
+                        timeleft = str(timedelta(seconds = nzb['RemainingSizeMB'] / status['DownloadRate'] * 2 ^ 20))
+                except:
+                    pass
+    
+                release_downloads.append({
+                    'id': nzb_id,
+                    'name': nzb['NZBFilename'],
+                    'original_status': 'DOWNLOADING' if nzb['ActiveDownloads'] > 0 else 'QUEUED',
+                    # Seems to have no native API function for time left. This will return the time left after NZBGet started downloading this item
+                    'timeleft': timeleft,
+                })
 
         for nzb in queue: # 'Parameters' is not passed in rpc.postqueue
-            log.debug('Found %s in NZBGet postprocessing queue', nzb['NZBFilename'])
-            release_downloads.append({
-                'id': nzb['NZBID'],
-                'name': nzb['NZBFilename'],
-                'original_status': nzb['Stage'],
-                'timeleft': str(timedelta(seconds = 0)) if not status['PostPaused'] else -1,
-            })
+            if nzb['NZBID'] in ids:
+                log.debug('Found %s in NZBGet postprocessing queue', nzb['NZBFilename'])
+                release_downloads.append({
+                    'id': nzb['NZBID'],
+                    'name': nzb['NZBFilename'],
+                    'original_status': nzb['Stage'],
+                    'timeleft': str(timedelta(seconds = 0)) if not status['PostPaused'] else -1,
+                })
 
         for nzb in history:
-            log.debug('Found %s in NZBGet history. ParStatus: %s, ScriptStatus: %s, Log: %s', (nzb['NZBFilename'] , nzb['ParStatus'], nzb['ScriptStatus'] , nzb['Log']))
             try:
                 nzb_id = [param['Value'] for param in nzb['Parameters'] if param['Name'] == 'couchpotato'][0]
             except:
                 nzb_id = nzb['NZBID']
-            release_downloads.append({
-                'id': nzb_id,
-                'name': nzb['NZBFilename'],
-                'status': 'completed' if nzb['ParStatus'] in ['SUCCESS', 'NONE'] and nzb['ScriptStatus'] in ['SUCCESS', 'NONE'] else 'failed',
-                'original_status': nzb['ParStatus'] + ', ' + nzb['ScriptStatus'],
-                'timeleft': str(timedelta(seconds = 0)),
-                'folder': sp(nzb['DestDir'])
-            })
+
+            if nzb_id in ids:
+                log.debug('Found %s in NZBGet history. ParStatus: %s, ScriptStatus: %s, Log: %s', (nzb['NZBFilename'] , nzb['ParStatus'], nzb['ScriptStatus'] , nzb['Log']))
+                release_downloads.append({
+                    'id': nzb_id,
+                    'name': nzb['NZBFilename'],
+                    'status': 'completed' if nzb['ParStatus'] in ['SUCCESS', 'NONE'] and nzb['ScriptStatus'] in ['SUCCESS', 'NONE'] else 'failed',
+                    'original_status': nzb['ParStatus'] + ', ' + nzb['ScriptStatus'],
+                    'timeleft': str(timedelta(seconds = 0)),
+                    'folder': sp(nzb['DestDir'])
+                })
 
         return release_downloads
 

couchpotato/core/downloaders/nzbvortex/main.py

@@ -8,9 +8,11 @@ from uuid import uuid4
 import hashlib
 import httplib
 import json
+import os
 import socket
 import ssl
 import sys
+import time
 import traceback
 import urllib2
 
@@ -23,44 +25,46 @@ class NZBVortex(Downloader):
     api_level = None
     session_id = None
 
-    def download(self, data = None, movie = None, filedata = None):
-        if not movie: movie = {}
+    def download(self, data = None, media = None, filedata = None):
+        if not media: media = {}
         if not data: data = {}
 
         # Send the nzb
         try:
-            nzb_filename = self.createFileName(data, filedata, movie)
+            nzb_filename = self.createFileName(data, filedata, media)
             self.call('nzb/add', params = {'file': (nzb_filename, filedata)}, multipart = True)
 
+            time.sleep(10)
             raw_statuses = self.call('nzb')
-            nzb_id = [nzb['id'] for nzb in raw_statuses.get('nzbs', []) if nzb['name'] == nzb_filename][0]
+            nzb_id = [nzb['id'] for nzb in raw_statuses.get('nzbs', []) if os.path.basename(item['nzbFileName']) == nzb_filename][0]
             return self.downloadReturnId(nzb_id)
         except:
             log.error('Something went wrong sending the NZB file: %s', traceback.format_exc())
             return False
 
-    def getAllDownloadStatus(self):
+    def getAllDownloadStatus(self, ids):
 
         raw_statuses = self.call('nzb')
 
         release_downloads = ReleaseDownloadList(self)
         for nzb in raw_statuses.get('nzbs', []):
+            if nzb['id'] in ids:
 
-            # Check status
-            status = 'busy'
-            if nzb['state'] == 20:
-                status = 'completed'
-            elif nzb['state'] in [21, 22, 24]:
-                status = 'failed'
-
-            release_downloads.append({
-                'id': nzb['id'],
-                'name': nzb['uiTitle'],
-                'status': status,
-                'original_status': nzb['state'],
-                'timeleft':-1,
-                'folder': sp(nzb['destinationPath']),
-            })
+                # Check status
+                status = 'busy'
+                if nzb['state'] == 20:
+                    status = 'completed'
+                elif nzb['state'] in [21, 22, 24]:
+                    status = 'failed'
+    
+                release_downloads.append({
+                    'id': nzb['id'],
+                    'name': nzb['uiTitle'],
+                    'status': status,
+                    'original_status': nzb['state'],
+                    'timeleft':-1,
+                    'folder': sp(nzb['destinationPath']),
+                })
 
         return release_downloads
 

couchpotato/core/downloaders/pneumatic/main.py

@@ -12,8 +12,8 @@ class Pneumatic(Downloader):
     protocol = ['nzb']
     strm_syntax = 'plugin://plugin.program.pneumatic/?mode=strm&type=add_file&nzb=%s&nzbname=%s'
 
-    def download(self, data = None, movie = None, filedata = None):
-        if not movie: movie = {}
+    def download(self, data = None, media = None, filedata = None):
+        if not media: media = {}
         if not data: data = {}
 
         directory = self.conf('directory')
@@ -25,7 +25,7 @@ class Pneumatic(Downloader):
                     log.error('No nzb available!')
                     return False
 
-                fullPath = os.path.join(directory, self.createFileName(data, filedata, movie))
+                fullPath = os.path.join(directory, self.createFileName(data, filedata, media))
 
                 try:
                     if not os.path.isfile(fullPath):
@@ -33,7 +33,7 @@ class Pneumatic(Downloader):
                         with open(fullPath, 'wb') as f:
                             f.write(filedata)
 
-                        nzb_name = self.createNzbName(data, movie)
+                        nzb_name = self.createNzbName(data, media)
                         strm_path = os.path.join(directory, nzb_name)
 
                         strm_file = open(strm_path + '.strm', 'wb')
@@ -41,11 +41,11 @@ class Pneumatic(Downloader):
                         strm_file.write(strmContent)
                         strm_file.close()
 
-                        return True
+                        return self.downloadReturnId('')
 
                     else:
                         log.info('File %s already exists.', fullPath)
-                        return True
+                        return self.downloadReturnId('')
 
                 except:
                     log.error('Failed to download .strm: %s', traceback.format_exc())

couchpotato/core/downloaders/rtorrent/__init__.py

@@ -58,14 +58,6 @@ config = [{
                     'advanced': True,
                     'description': 'Also remove the leftover files.',
                 },
-                {
-                    'name': 'append_label',
-                    'label': 'Append Label',
-                    'default': False,
-                    'advanced': True,
-                    'type': 'bool',
-                    'description': 'Append label to download location. Requires you to set the download location above.',
-                },
                 {
                     'name': 'paused',
                     'type': 'bool',

couchpotato/core/downloaders/rtorrent/main.py

@@ -77,7 +77,10 @@ class rTorrent(Downloader):
         return True
 
 
-    def download(self, data, movie, filedata = None):
+    def download(self, data = None, media = None, filedata = None):
+        if not media: media = {}
+        if not data: data = {}
+
         log.debug('Sending "%s" to rTorrent.', (data.get('name')))
 
         if not self.connect():
@@ -125,9 +128,7 @@ class rTorrent(Downloader):
             if self.conf('label'):
                 torrent.set_custom(1, self.conf('label'))
 
-            if self.conf('directory') and self.conf('append_label'):
-                torrent.set_directory(os.path.join(self.conf('directory'), self.conf('label')))
-            elif self.conf('directory'):
+            if self.conf('directory'):
                 torrent.set_directory(self.conf('directory'))
 
             # Set Ratio Group
@@ -142,7 +143,7 @@ class rTorrent(Downloader):
             log.error('Failed to send torrent to rTorrent: %s', err)
             return False
 
-    def getAllDownloadStatus(self):
+    def getAllDownloadStatus(self, ids):
         log.debug('Checking rTorrent download status.')
 
         if not self.connect():
@@ -154,27 +155,28 @@ class rTorrent(Downloader):
             release_downloads = ReleaseDownloadList(self)
 
             for torrent in torrents:
-                torrent_files = []
-                for file_item in torrent.get_files():
-                    torrent_files.append(sp(os.path.join(torrent.directory, file_item.path)))
-
-                status = 'busy'
-                if torrent.complete:
-                    if torrent.active:
-                        status = 'seeding'
-                    else:
-                        status = 'completed'
-
-                release_downloads.append({
-                    'id': torrent.info_hash,
-                    'name': torrent.name,
-                    'status': status,
-                    'seed_ratio': torrent.ratio,
-                    'original_status': torrent.state,
-                    'timeleft': str(timedelta(seconds = float(torrent.left_bytes) / torrent.down_rate)) if torrent.down_rate > 0 else -1,
-                    'folder': sp(torrent.directory),
-                    'files': '|'.join(torrent_files)
-                })
+                if torrent.info_hash in ids:
+                    torrent_files = []
+                    for file_item in torrent.get_files():
+                        torrent_files.append(sp(os.path.join(torrent.directory, file_item.path)))
+    
+                    status = 'busy'
+                    if torrent.complete:
+                        if torrent.active:
+                            status = 'seeding'
+                        else:
+                            status = 'completed'
+    
+                    release_downloads.append({
+                        'id': torrent.info_hash,
+                        'name': torrent.name,
+                        'status': status,
+                        'seed_ratio': torrent.ratio,
+                        'original_status': torrent.state,
+                        'timeleft': str(timedelta(seconds = float(torrent.left_bytes) / torrent.down_rate)) if torrent.down_rate > 0 else -1,
+                        'folder': sp(torrent.directory),
+                        'files': '|'.join(torrent_files)
+                    })
 
             return release_downloads
 

couchpotato/core/downloaders/sabnzbd/main.py

@@ -16,8 +16,8 @@ class Sabnzbd(Downloader):
 
     protocol = ['nzb']
 
-    def download(self, data = None, movie = None, filedata = None):
-        if not movie: movie = {}
+    def download(self, data = None, media = None, filedata = None):
+        if not media: media = {}
         if not data: data = {}
 
         log.info('Sending "%s" to SABnzbd.', data.get('name'))
@@ -25,7 +25,7 @@ class Sabnzbd(Downloader):
         req_params = {
             'cat': self.conf('category'),
             'mode': 'addurl',
-            'nzbname': self.createNzbName(data, movie),
+            'nzbname': self.createNzbName(data, media),
             'priority': self.conf('priority'),
         }
 
@@ -36,7 +36,7 @@ class Sabnzbd(Downloader):
                 return False
 
             # If it's a .rar, it adds the .rar extension, otherwise it stays .nzb
-            nzb_filename = self.createFileName(data, filedata, movie)
+            nzb_filename = self.createFileName(data, filedata, media)
             req_params['mode'] = 'addfile'
         else:
             req_params['name'] = data.get('url')
@@ -64,7 +64,7 @@ class Sabnzbd(Downloader):
             log.error('Error getting data from SABNZBd: %s', sab_data)
             return False
 
-    def getAllDownloadStatus(self):
+    def getAllDownloadStatus(self, ids):
 
         log.debug('Checking SABnzbd download status.')
 
@@ -91,35 +91,36 @@ class Sabnzbd(Downloader):
 
         # Get busy releases
         for nzb in queue.get('slots', []):
-            status = 'busy'
-            if 'ENCRYPTED / ' in nzb['filename']:
-                status = 'failed'
-
-            release_downloads.append({
-                'id': nzb['nzo_id'],
-                'name': nzb['filename'],
-                'status': status,
-                'original_status': nzb['status'],
-                'timeleft': nzb['timeleft'] if not queue['paused'] else -1,
-            })
+            if nzb['nzo_id'] in ids:
+                status = 'busy'
+                if 'ENCRYPTED / ' in nzb['filename']:
+                    status = 'failed'
+    
+                release_downloads.append({
+                    'id': nzb['nzo_id'],
+                    'name': nzb['filename'],
+                    'status': status,
+                    'original_status': nzb['status'],
+                    'timeleft': nzb['timeleft'] if not queue['paused'] else -1,
+                })
 
         # Get old releases
         for nzb in history.get('slots', []):
-
-            status = 'busy'
-            if nzb['status'] == 'Failed' or (nzb['status'] == 'Completed' and nzb['fail_message'].strip()):
-                status = 'failed'
-            elif nzb['status'] == 'Completed':
-                status = 'completed'
-
-            release_downloads.append({
-                'id': nzb['nzo_id'],
-                'name': nzb['name'],
-                'status': status,
-                'original_status': nzb['status'],
-                'timeleft': str(timedelta(seconds = 0)),
-                'folder': sp(os.path.dirname(nzb['storage']) if os.path.isfile(nzb['storage']) else nzb['storage']),
-            })
+            if nzb['nzo_id'] in ids:
+                status = 'busy'
+                if nzb['status'] == 'Failed' or (nzb['status'] == 'Completed' and nzb['fail_message'].strip()):
+                    status = 'failed'
+                elif nzb['status'] == 'Completed':
+                    status = 'completed'
+    
+                release_downloads.append({
+                    'id': nzb['nzo_id'],
+                    'name': nzb['name'],
+                    'status': status,
+                    'original_status': nzb['status'],
+                    'timeleft': str(timedelta(seconds = 0)),
+                    'folder': sp(os.path.dirname(nzb['storage']) if os.path.isfile(nzb['storage']) else nzb['storage']),
+                })
 
         return release_downloads
 

couchpotato/core/downloaders/synology/main.py

@@ -3,6 +3,7 @@ from couchpotato.core.helpers.encoding import isInt
 from couchpotato.core.logger import CPLog
 import json
 import requests
+import traceback
 
 log = CPLog(__name__)
 
@@ -12,8 +13,8 @@ class Synology(Downloader):
     protocol = ['nzb', 'torrent', 'torrent_magnet']
     log = CPLog(__name__)
 
-    def download(self, data = None, movie = None, filedata = None):
-        if not movie: movie = {}
+    def download(self, data = None, media = None, filedata = None):
+        if not media: media = {}
         if not data: data = {}
 
         response = False
@@ -34,14 +35,14 @@ class Synology(Downloader):
             elif data['protocol'] in ['nzb', 'torrent']:
                 log.info('Adding %s' % data['protocol'])
                 if not filedata:
-                    log.error('No %s data found' % data['protocol'])
+                    log.error('No %s data found', data['protocol'])
                 else:
                     filename = data['name'] + '.' + data['protocol']
                     response = srpc.create_task(filename = filename, filedata = filedata)
-        except Exception, err:
-            log.error('Exception while adding torrent: %s', err)
+        except:
+            log.error('Exception while adding torrent: %s', traceback.format_exc())
         finally:
-            return response
+            return self.downloadReturnId('') if response else False
 
     def getEnabledProtocol(self):
         if self.conf('use_for') == 'both':

couchpotato/core/downloaders/transmission/main.py

@@ -31,7 +31,9 @@ class Transmission(Downloader):
 
         return self.trpc
 
-    def download(self, data, movie, filedata = None):
+    def download(self, data = None, media = None, filedata = None):
+        if not media: media = {}
+        if not data: data = {}
 
         log.info('Sending "%s" (%s) to Transmission.', (data.get('name'), data.get('protocol')))
 
@@ -81,7 +83,7 @@ class Transmission(Downloader):
         log.info('Torrent sent to Transmission successfully.')
         return self.downloadReturnId(remote_torrent['torrent-added']['hashString'])
 
-    def getAllDownloadStatus(self):
+    def getAllDownloadStatus(self, ids):
 
         log.debug('Checking Transmission download status.')
 
@@ -100,31 +102,32 @@ class Transmission(Downloader):
             return False
 
         for torrent in queue['torrents']:
-            log.debug('name=%s / id=%s / downloadDir=%s / hashString=%s / percentDone=%s / status=%s / eta=%s / uploadRatio=%s / isFinished=%s',
-                (torrent['name'], torrent['id'], torrent['downloadDir'], torrent['hashString'], torrent['percentDone'], torrent['status'], torrent['eta'], torrent['uploadRatio'], torrent['isFinished']))
-
-            torrent_files = []
-            for file_item in torrent['files']:
-                torrent_files.append(sp(os.path.join(torrent['downloadDir'], file_item['name'])))
-
-            status = 'busy'
-            if torrent.get('isStalled') and self.conf('stalled_as_failed'):
-                status = 'failed'
-            elif torrent['status'] == 0 and torrent['percentDone'] == 1:
-                status = 'completed'
-            elif torrent['status'] in [5, 6]:
-                status = 'seeding'
-
-            release_downloads.append({
-                'id': torrent['hashString'],
-                'name': torrent['name'],
-                'status': status,
-                'original_status': torrent['status'],
-                'seed_ratio': torrent['uploadRatio'],
-                'timeleft': str(timedelta(seconds = torrent['eta'])),
-                'folder': sp(torrent['downloadDir'] if len(torrent_files) == 1 else os.path.join(torrent['downloadDir'], torrent['name'])),
-                'files': '|'.join(torrent_files)
-            })
+            if torrent['hashString'] in ids:
+                log.debug('name=%s / id=%s / downloadDir=%s / hashString=%s / percentDone=%s / status=%s / eta=%s / uploadRatio=%s / isFinished=%s',
+                    (torrent['name'], torrent['id'], torrent['downloadDir'], torrent['hashString'], torrent['percentDone'], torrent['status'], torrent['eta'], torrent['uploadRatio'], torrent['isFinished']))
+    
+                torrent_files = []
+                for file_item in torrent['files']:
+                    torrent_files.append(sp(os.path.join(torrent['downloadDir'], file_item['name'])))
+    
+                status = 'busy'
+                if torrent.get('isStalled') and self.conf('stalled_as_failed'):
+                    status = 'failed'
+                elif torrent['status'] == 0 and torrent['percentDone'] == 1:
+                    status = 'completed'
+                elif torrent['status'] in [5, 6]:
+                    status = 'seeding'
+    
+                release_downloads.append({
+                    'id': torrent['hashString'],
+                    'name': torrent['name'],
+                    'status': status,
+                    'original_status': torrent['status'],
+                    'seed_ratio': torrent['uploadRatio'],
+                    'timeleft': str(timedelta(seconds = torrent['eta'])),
+                    'folder': sp(torrent['downloadDir'] if len(torrent_files) == 1 else os.path.join(torrent['downloadDir'], torrent['name'])),
+                    'files': '|'.join(torrent_files)
+                })
 
         return release_downloads
 

couchpotato/core/downloaders/utorrent/main.py

@@ -36,8 +36,8 @@ class uTorrent(Downloader):
 
         return self.utorrent_api
 
-    def download(self, data = None, movie = None, filedata = None):
-        if not movie: movie = {}
+    def download(self, data = None, media = None, filedata = None):
+        if not media: media = {}
         if not data: data = {}
 
         log.debug('Sending "%s" (%s) to uTorrent.', (data.get('name'), data.get('protocol')))
@@ -77,7 +77,8 @@ class uTorrent(Downloader):
         else:
             info = bdecode(filedata)["info"]
             torrent_hash = sha1(benc(info)).hexdigest().upper()
-        torrent_filename = self.createFileName(data, filedata, movie)
+
+        torrent_filename = self.createFileName(data, filedata, media)
 
         if data.get('seed_ratio'):
             torrent_params['seed_override'] = 1
@@ -104,7 +105,7 @@ class uTorrent(Downloader):
 
         return self.downloadReturnId(torrent_hash)
 
-    def getAllDownloadStatus(self):
+    def getAllDownloadStatus(self, ids):
 
         log.debug('Checking uTorrent download status.')
 
@@ -129,47 +130,48 @@ class uTorrent(Downloader):
 
         # Get torrents
         for torrent in queue['torrents']:
+            if torrent[0] in ids:
 
-            #Get files of the torrent
-            torrent_files = []
-            try:
-                torrent_files = json.loads(self.utorrent_api.get_files(torrent[0]))
-                torrent_files = [sp(os.path.join(torrent[26], torrent_file[0])) for torrent_file in torrent_files['files'][1]]
-            except:
-                log.debug('Failed getting files from torrent: %s', torrent[2])
-
-            status_flags = {
-                "STARTED"     : 1,
-                "CHECKING"    : 2,
-                "CHECK-START" : 4,
-                "CHECKED"     : 8,
-                "ERROR"       : 16,
-                "PAUSED"      : 32,
-                "QUEUED"      : 64,
-                "LOADED"      : 128
-            }
-
-            status = 'busy'
-            if (torrent[1] & status_flags["STARTED"] or torrent[1] & status_flags["QUEUED"]) and torrent[4] == 1000:
-                status = 'seeding'
-            elif (torrent[1] & status_flags["ERROR"]):
-                status = 'failed'
-            elif torrent[4] == 1000:
-                status = 'completed'
-
-            if not status == 'busy':
-                self.removeReadOnly(torrent_files)
-
-            release_downloads.append({
-                'id': torrent[0],
-                'name': torrent[2],
-                'status': status,
-                'seed_ratio': float(torrent[7]) / 1000,
-                'original_status': torrent[1],
-                'timeleft': str(timedelta(seconds = torrent[10])),
-                'folder': sp(torrent[26]),
-                'files': '|'.join(torrent_files)
-            })
+                #Get files of the torrent
+                torrent_files = []
+                try:
+                    torrent_files = json.loads(self.utorrent_api.get_files(torrent[0]))
+                    torrent_files = [sp(os.path.join(torrent[26], torrent_file[0])) for torrent_file in torrent_files['files'][1]]
+                except:
+                    log.debug('Failed getting files from torrent: %s', torrent[2])
+    
+                status_flags = {
+                    "STARTED"     : 1,
+                    "CHECKING"    : 2,
+                    "CHECK-START" : 4,
+                    "CHECKED"     : 8,
+                    "ERROR"       : 16,
+                    "PAUSED"      : 32,
+                    "QUEUED"      : 64,
+                    "LOADED"      : 128
+                }
+    
+                status = 'busy'
+                if (torrent[1] & status_flags["STARTED"] or torrent[1] & status_flags["QUEUED"]) and torrent[4] == 1000:
+                    status = 'seeding'
+                elif (torrent[1] & status_flags["ERROR"]):
+                    status = 'failed'
+                elif torrent[4] == 1000:
+                    status = 'completed'
+    
+                if not status == 'busy':
+                    self.removeReadOnly(torrent_files)
+    
+                release_downloads.append({
+                    'id': torrent[0],
+                    'name': torrent[2],
+                    'status': status,
+                    'seed_ratio': float(torrent[7]) / 1000,
+                    'original_status': torrent[1],
+                    'timeleft': str(timedelta(seconds = torrent[10])),
+                    'folder': sp(torrent[26]),
+                    'files': '|'.join(torrent_files)
+                })
 
         return release_downloads
 

couchpotato/core/helpers/encoding.py

@@ -49,8 +49,21 @@ def ss(original, *args):
         return u_original.encode('UTF-8')
 
 def sp(path, *args):
+
     # Standardise encoding, normalise case, path and strip trailing '/' or '\'
-    return os.path.normcase(os.path.normpath(ss(path, *args))).rstrip(os.path.sep)
+    if not path or len(path) == 0:
+        return path
+
+    # convert windows path (from remote box) to *nix path
+    if os.path.sep == '/' and '\\' in path:
+        path = '/' + path.replace(':', '').replace('\\', '/')
+
+    path = os.path.normcase(os.path.normpath(ss(path, *args)))
+
+    if path != os.path.sep:
+        path = path.rstrip(os.path.sep)
+
+    return path
 
 def ek(original, *args):
     if isinstance(original, (str, unicode)):

couchpotato/core/helpers/variable.py

@@ -11,6 +11,9 @@ import sys
 
 log = CPLog(__name__)
 
+def fnEscape(pattern):
+    return pattern.replace('[','[[').replace(']','[]]').replace('[[','[[]')
+
 def link(src, dst):
     if os.name == 'nt':
         import ctypes

couchpotato/core/media/__init__.py

@@ -38,7 +38,7 @@ class MediaBase(Plugin):
         def notifyFront():
             db = get_session()
             media = db.query(Media).filter_by(id = media_id).first()
-            fireEvent('notify.frontend', type = '%s.update.%s' % (media.type, media.id), data = media.to_dict(self.default_dict))
+            fireEvent('notify.frontend', type = '%s.update' % media.type, data = media.to_dict(self.default_dict))
             db.expire_all()
 
         return notifyFront

couchpotato/core/media/_base/media/main.py

@@ -1,10 +1,15 @@
 from couchpotato import get_session
 from couchpotato.api import addApiView
 from couchpotato.core.event import fireEvent, fireEventAsync, addEvent
-from couchpotato.core.helpers.variable import splitString
+from couchpotato.core.helpers.encoding import toUnicode
+from couchpotato.core.helpers.variable import mergeDicts, splitString, getImdb
 from couchpotato.core.logger import CPLog
 from couchpotato.core.media import MediaBase
-from couchpotato.core.settings.model import Media
+from couchpotato.core.settings.model import Library, LibraryTitle, Release, \
+    Media
+from sqlalchemy.orm import joinedload_all
+from sqlalchemy.sql.expression import or_, asc, not_, desc
+from string import ascii_lowercase
 
 log = CPLog(__name__)
 
@@ -20,7 +25,49 @@ class MediaPlugin(MediaBase):
             }
         })
 
-        addEvent('app.load', self.addSingleRefresh)
+        addApiView('media.list', self.listView, docs = {
+            'desc': 'List media',
+            'params': {
+                'type': {'type': 'string', 'desc': 'Media type to filter on.'},
+                'status': {'type': 'array or csv', 'desc': 'Filter movie by status. Example:"active,done"'},
+                'release_status': {'type': 'array or csv', 'desc': 'Filter movie by status of its releases. Example:"snatched,available"'},
+                'limit_offset': {'desc': 'Limit and offset the movie list. Examples: "50" or "50,30"'},
+                'starts_with': {'desc': 'Starts with these characters. Example: "a" returns all movies starting with the letter "a"'},
+                'search': {'desc': 'Search movie title'},
+            },
+            'return': {'type': 'object', 'example': """{
+    'success': True,
+    'empty': bool, any movies returned or not,
+    'media': array, media found,
+}"""}
+        })
+
+        addApiView('media.get', self.getView, docs = {
+            'desc': 'Get media by id',
+            'params': {
+                'id': {'desc': 'The id of the media'},
+            }
+        })
+
+        addApiView('media.delete', self.deleteView, docs = {
+            'desc': 'Delete a media from the wanted list',
+            'params': {
+                'id': {'desc': 'Media ID(s) you want to delete.', 'type': 'int (comma separated)'},
+                'delete_from': {'desc': 'Delete media from this page', 'type': 'string: all (default), wanted, manage'},
+            }
+        })
+
+        addApiView('media.available_chars', self.charView)
+
+        addEvent('app.load', self.addSingleRefreshView)
+        addEvent('app.load', self.addSingleListView)
+        addEvent('app.load', self.addSingleCharView)
+        addEvent('app.load', self.addSingleDeleteView)
+
+        addEvent('media.get', self.get)
+        addEvent('media.list', self.list)
+        addEvent('media.delete', self.delete)
+        addEvent('media.restatus', self.restatus)
 
     def refresh(self, id = '', **kwargs):
         db = get_session()
@@ -34,7 +81,7 @@ class MediaPlugin(MediaBase):
                 for title in media.library.titles:
                     if title.default: default_title = title.title
 
-                fireEvent('notify.frontend', type = '%s.busy.%s' % (media.type, x), data = True)
+                fireEvent('notify.frontend', type = '%s.busy' % media.type, data = {'id': x})
                 fireEventAsync('library.update.%s' % media.type, identifier = media.library.identifier, default_title = default_title, force = True, on_complete = self.createOnComplete(x))
 
         db.expire_all()
@@ -43,7 +90,369 @@ class MediaPlugin(MediaBase):
             'success': True,
         }
 
-    def addSingleRefresh(self):
+    def addSingleRefreshView(self):
 
         for media_type in fireEvent('media.types', merge = True):
             addApiView('%s.refresh' % media_type, self.refresh)
+
+    def get(self, media_id):
+
+        db = get_session()
+
+        imdb_id = getImdb(str(media_id))
+
+        if imdb_id:
+            m = db.query(Media).filter(Media.library.has(identifier = imdb_id)).first()
+        else:
+            m = db.query(Media).filter_by(id = media_id).first()
+
+        results = None
+        if m:
+            results = m.to_dict(self.default_dict)
+
+        db.expire_all()
+        return results
+
+    def getView(self, id = None, **kwargs):
+
+        media = self.get(id) if id else None
+
+        return {
+            'success': media is not None,
+            'media': media,
+        }
+
+    def list(self, types = None, status = None, release_status = None, limit_offset = None, starts_with = None, search = None, order = None):
+
+        db = get_session()
+
+        # Make a list from string
+        if status and not isinstance(status, (list, tuple)):
+            status = [status]
+        if release_status and not isinstance(release_status, (list, tuple)):
+            release_status = [release_status]
+        if types and not isinstance(types, (list, tuple)):
+            types = [types]
+
+        # query movie ids
+        q = db.query(Media) \
+            .with_entities(Media.id) \
+            .group_by(Media.id)
+
+        # Filter on movie status
+        if status and len(status) > 0:
+            statuses = fireEvent('status.get', status, single = len(status) > 1)
+            statuses = [s.get('id') for s in statuses]
+
+            q = q.filter(Media.status_id.in_(statuses))
+
+        # Filter on release status
+        if release_status and len(release_status) > 0:
+            q = q.join(Media.releases)
+
+            statuses = fireEvent('status.get', release_status, single = len(release_status) > 1)
+            statuses = [s.get('id') for s in statuses]
+
+            q = q.filter(Release.status_id.in_(statuses))
+
+        # Filter on type
+        if types and len(types) > 0:
+            try: q = q.filter(Media.type.in_(types))
+            except: pass
+
+        # Only join when searching / ordering
+        if starts_with or search or order != 'release_order':
+            q = q.join(Media.library, Library.titles) \
+                .filter(LibraryTitle.default == True)
+
+        # Add search filters
+        filter_or = []
+        if starts_with:
+            starts_with = toUnicode(starts_with.lower())
+            if starts_with in ascii_lowercase:
+                filter_or.append(LibraryTitle.simple_title.startswith(starts_with))
+            else:
+                ignore = []
+                for letter in ascii_lowercase:
+                    ignore.append(LibraryTitle.simple_title.startswith(toUnicode(letter)))
+                filter_or.append(not_(or_(*ignore)))
+
+        if search:
+            filter_or.append(LibraryTitle.simple_title.like('%%' + search + '%%'))
+
+        if len(filter_or) > 0:
+            q = q.filter(or_(*filter_or))
+
+        total_count = q.count()
+        if total_count == 0:
+            return 0, []
+
+        if order == 'release_order':
+            q = q.order_by(desc(Release.last_edit))
+        else:
+            q = q.order_by(asc(LibraryTitle.simple_title))
+
+        if limit_offset:
+            splt = splitString(limit_offset) if isinstance(limit_offset, (str, unicode)) else limit_offset
+            limit = splt[0]
+            offset = 0 if len(splt) is 1 else splt[1]
+            q = q.limit(limit).offset(offset)
+
+        # Get all media_ids in sorted order
+        media_ids = [m.id for m in q.all()]
+
+        # List release statuses
+        releases = db.query(Release) \
+            .filter(Release.movie_id.in_(media_ids)) \
+            .all()
+
+        release_statuses = dict((m, set()) for m in media_ids)
+        releases_count = dict((m, 0) for m in media_ids)
+        for release in releases:
+            release_statuses[release.movie_id].add('%d,%d' % (release.status_id, release.quality_id))
+            releases_count[release.movie_id] += 1
+
+        # Get main movie data
+        q2 = db.query(Media) \
+            .options(joinedload_all('library.titles')) \
+            .options(joinedload_all('library.files')) \
+            .options(joinedload_all('status')) \
+            .options(joinedload_all('files'))
+
+        q2 = q2.filter(Media.id.in_(media_ids))
+
+        results = q2.all()
+
+        # Create dict by movie id
+        movie_dict = {}
+        for movie in results:
+            movie_dict[movie.id] = movie
+
+        # List movies based on media_ids order
+        movies = []
+        for media_id in media_ids:
+
+            releases = []
+            for r in release_statuses.get(media_id):
+                x = splitString(r)
+                releases.append({'status_id': x[0], 'quality_id': x[1]})
+
+            # Merge releases with movie dict
+            movies.append(mergeDicts(movie_dict[media_id].to_dict({
+                'library': {'titles': {}, 'files':{}},
+                'files': {},
+            }), {
+                'releases': releases,
+                'releases_count': releases_count.get(media_id),
+            }))
+
+        db.expire_all()
+        return total_count, movies
+
+    def listView(self, **kwargs):
+
+        types = splitString(kwargs.get('types'))
+        status = splitString(kwargs.get('status'))
+        release_status = splitString(kwargs.get('release_status'))
+        limit_offset = kwargs.get('limit_offset')
+        starts_with = kwargs.get('starts_with')
+        search = kwargs.get('search')
+        order = kwargs.get('order')
+
+        total_movies, movies = self.list(
+            types = types,
+            status = status,
+            release_status = release_status,
+            limit_offset = limit_offset,
+            starts_with = starts_with,
+            search = search,
+            order = order
+        )
+
+        return {
+            'success': True,
+            'empty': len(movies) == 0,
+            'total': total_movies,
+            'movies': movies,
+        }
+
+    def addSingleListView(self):
+
+        for media_type in fireEvent('media.types', merge = True):
+            def tempList(*args, **kwargs):
+                return self.listView(types = media_type, *args, **kwargs)
+            addApiView('%s.list' % media_type, tempList)
+
+    def availableChars(self, types = None, status = None, release_status = None):
+
+        types = types or []
+        status = status or []
+        release_status = release_status or []
+
+        db = get_session()
+
+        # Make a list from string
+        if not isinstance(status, (list, tuple)):
+            status = [status]
+        if release_status and not isinstance(release_status, (list, tuple)):
+            release_status = [release_status]
+        if types and not isinstance(types, (list, tuple)):
+            types = [types]
+
+        q = db.query(Media)
+
+        # Filter on movie status
+        if status and len(status) > 0:
+            statuses = fireEvent('status.get', status, single = len(release_status) > 1)
+            statuses = [s.get('id') for s in statuses]
+
+            q = q.filter(Media.status_id.in_(statuses))
+
+        # Filter on release status
+        if release_status and len(release_status) > 0:
+
+            statuses = fireEvent('status.get', release_status, single = len(release_status) > 1)
+            statuses = [s.get('id') for s in statuses]
+
+            q = q.join(Media.releases) \
+                .filter(Release.status_id.in_(statuses))
+
+        # Filter on type
+        if types and len(types) > 0:
+            try: q = q.filter(Media.type.in_(types))
+            except: pass
+
+        q = q.join(Library, LibraryTitle) \
+            .with_entities(LibraryTitle.simple_title) \
+            .filter(LibraryTitle.default == True)
+
+        titles = q.all()
+
+        chars = set()
+        for title in titles:
+            try:
+                char = title[0][0]
+                char = char if char in ascii_lowercase else '#'
+                chars.add(str(char))
+            except:
+                log.error('Failed getting title for %s', title.libraries_id)
+
+            if len(chars) == 25:
+                break
+
+        db.expire_all()
+        return ''.join(sorted(chars))
+
+    def charView(self, **kwargs):
+
+        type = splitString(kwargs.get('type', 'movie'))
+        status = splitString(kwargs.get('status', None))
+        release_status = splitString(kwargs.get('release_status', None))
+        chars = self.availableChars(type, status, release_status)
+
+        return {
+            'success': True,
+            'empty': len(chars) == 0,
+            'chars': chars,
+        }
+
+    def addSingleCharView(self):
+
+        for media_type in fireEvent('media.types', merge = True):
+            def tempChar(*args, **kwargs):
+                return self.charView(types = media_type, *args, **kwargs)
+            addApiView('%s.available_chars' % media_type, tempChar)
+
+    def delete(self, media_id, delete_from = None):
+
+        db = get_session()
+
+        media = db.query(Media).filter_by(id = media_id).first()
+        if media:
+            deleted = False
+            if delete_from == 'all':
+                db.delete(media)
+                db.commit()
+                deleted = True
+            else:
+                done_status = fireEvent('status.get', 'done', single = True)
+
+                total_releases = len(media.releases)
+                total_deleted = 0
+                new_movie_status = None
+                for release in media.releases:
+                    if delete_from in ['wanted', 'snatched', 'late']:
+                        if release.status_id != done_status.get('id'):
+                            db.delete(release)
+                            total_deleted += 1
+                        new_movie_status = 'done'
+                    elif delete_from == 'manage':
+                        if release.status_id == done_status.get('id'):
+                            db.delete(release)
+                            total_deleted += 1
+                        new_movie_status = 'active'
+                db.commit()
+
+                if total_releases == total_deleted:
+                    db.delete(media)
+                    db.commit()
+                    deleted = True
+                elif new_movie_status:
+                    new_status = fireEvent('status.get', new_movie_status, single = True)
+                    media.profile_id = None
+                    media.status_id = new_status.get('id')
+                    db.commit()
+                else:
+                    fireEvent('media.restatus', media.id, single = True)
+
+            if deleted:
+                fireEvent('notify.frontend', type = 'movie.deleted', data = media.to_dict())
+
+        db.expire_all()
+        return True
+
+    def deleteView(self, id = '', **kwargs):
+
+        ids = splitString(id)
+        for media_id in ids:
+            self.delete(media_id, delete_from = kwargs.get('delete_from', 'all'))
+
+        return {
+            'success': True,
+        }
+
+    def addSingleDeleteView(self):
+
+        for media_type in fireEvent('media.types', merge = True):
+            def tempDelete(*args, **kwargs):
+                return self.deleteView(types = media_type, *args, **kwargs)
+            addApiView('%s.delete' % media_type, tempDelete)
+
+    def restatus(self, media_id):
+
+        active_status, done_status = fireEvent('status.get', ['active', 'done'], single = True)
+
+        db = get_session()
+
+        m = db.query(Media).filter_by(id = media_id).first()
+        if not m or len(m.library.titles) == 0:
+            log.debug('Can\'t restatus movie, doesn\'t seem to exist.')
+            return False
+
+        log.debug('Changing status for %s', m.library.titles[0].title)
+        if not m.profile:
+            m.status_id = done_status.get('id')
+        else:
+            move_to_wanted = True
+
+            for t in m.profile.types:
+                for release in m.releases:
+                    if t.quality.identifier is release.quality.identifier and (release.status_id is done_status.get('id') and t.finish):
+                        move_to_wanted = False
+
+            m.status_id = active_status.get('id') if move_to_wanted else done_status.get('id')
+
+        db.commit()
+
+        return True
+

couchpotato/core/media/_base/searcher/main.py

@@ -147,7 +147,7 @@ class Searcher(SearcherBase):
         except: pass
 
         # Match longest name between []
-        try: check_names.append(max(check_name.split('['), key = len))
+        try: check_names.append(max(re.findall(r'[^[]*\[([^]]*)\]', check_name), key = len).strip())
         except: pass
 
         for check_name in list(set(check_names)):

couchpotato/core/media/movie/_base/main.py

@@ -2,15 +2,10 @@ from couchpotato import get_session
 from couchpotato.api import addApiView
 from couchpotato.core.event import fireEvent, fireEventAsync, addEvent
 from couchpotato.core.helpers.encoding import toUnicode
-from couchpotato.core.helpers.variable import getImdb, splitString, tryInt, \
-    mergeDicts
+from couchpotato.core.helpers.variable import splitString, tryInt
 from couchpotato.core.logger import CPLog
 from couchpotato.core.media.movie import MovieTypeBase
-from couchpotato.core.settings.model import Library, LibraryTitle, Media, \
-    Release
-from sqlalchemy.orm import joinedload_all
-from sqlalchemy.sql.expression import or_, asc, not_, desc
-from string import ascii_lowercase
+from couchpotato.core.settings.model import Media
 import time
 
 log = CPLog(__name__)
@@ -26,33 +21,12 @@ class MovieBase(MovieTypeBase):
         super(MovieBase, self).__init__()
         self.initType()
 
-        addApiView('movie.list', self.listView, docs = {
-            'desc': 'List movies in wanted list',
-            'params': {
-                'status': {'type': 'array or csv', 'desc': 'Filter movie by status. Example:"active,done"'},
-                'release_status': {'type': 'array or csv', 'desc': 'Filter movie by status of its releases. Example:"snatched,available"'},
-                'limit_offset': {'desc': 'Limit and offset the movie list. Examples: "50" or "50,30"'},
-                'starts_with': {'desc': 'Starts with these characters. Example: "a" returns all movies starting with the letter "a"'},
-                'search': {'desc': 'Search movie title'},
-            },
-            'return': {'type': 'object', 'example': """{
-    'success': True,
-    'empty': bool, any movies returned or not,
-    'movies': array, movies found,
-}"""}
-        })
-        addApiView('movie.get', self.getView, docs = {
-            'desc': 'Get a movie by id',
-            'params': {
-                'id': {'desc': 'The id of the movie'},
-            }
-        })
-        addApiView('movie.available_chars', self.charView)
         addApiView('movie.add', self.addView, docs = {
             'desc': 'Add new movie to the wanted list',
             'params': {
                 'identifier': {'desc': 'IMDB id of the movie your want to add.'},
                 'profile_id': {'desc': 'ID of quality profile you want the add the movie in. If empty will use the default profile.'},
+                'category_id': {'desc': 'ID of category you want the add the movie in. If empty will use no category.'},
                 'title': {'desc': 'Movie title to use for searches. Has to be one of the titles returned by movie.search.'},
             }
         })
@@ -64,255 +38,8 @@ class MovieBase(MovieTypeBase):
                 'default_title': {'desc': 'Movie title to use for searches. Has to be one of the titles returned by movie.search.'},
             }
         })
-        addApiView('movie.delete', self.deleteView, docs = {
-            'desc': 'Delete a movie from the wanted list',
-            'params': {
-                'id': {'desc': 'Movie ID(s) you want to delete.', 'type': 'int (comma separated)'},
-                'delete_from': {'desc': 'Delete movie from this page', 'type': 'string: all (default), wanted, manage'},
-            }
-        })
 
         addEvent('movie.add', self.add)
-        addEvent('movie.delete', self.delete)
-        addEvent('movie.get', self.get)
-        addEvent('movie.list', self.list)
-        addEvent('movie.restatus', self.restatus)
-
-    def getView(self, id = None, **kwargs):
-
-        movie = self.get(id) if id else None
-
-        return {
-            'success': movie is not None,
-            'movie': movie,
-        }
-
-    def get(self, movie_id):
-
-        db = get_session()
-
-        imdb_id = getImdb(str(movie_id))
-
-        if imdb_id:
-            m = db.query(Media).filter(Media.library.has(identifier = imdb_id)).first()
-        else:
-            m = db.query(Media).filter_by(id = movie_id).first()
-
-        results = None
-        if m:
-            results = m.to_dict(self.default_dict)
-
-        db.expire_all()
-        return results
-
-    def list(self, status = None, release_status = None, limit_offset = None, starts_with = None, search = None, order = None):
-
-        db = get_session()
-
-        # Make a list from string
-        if status and not isinstance(status, (list, tuple)):
-            status = [status]
-        if release_status and not isinstance(release_status, (list, tuple)):
-            release_status = [release_status]
-
-        # query movie ids
-        q = db.query(Media) \
-            .with_entities(Media.id) \
-            .group_by(Media.id)
-
-        # Filter on movie status
-        if status and len(status) > 0:
-            statuses = fireEvent('status.get', status, single = len(status) > 1)
-            statuses = [s.get('id') for s in statuses]
-
-            q = q.filter(Media.status_id.in_(statuses))
-
-        # Filter on release status
-        if release_status and len(release_status) > 0:
-            q = q.join(Media.releases)
-
-            statuses = fireEvent('status.get', release_status, single = len(release_status) > 1)
-            statuses = [s.get('id') for s in statuses]
-
-            q = q.filter(Release.status_id.in_(statuses))
-
-        # Only join when searching / ordering
-        if starts_with or search or order != 'release_order':
-            q = q.join(Media.library, Library.titles) \
-                .filter(LibraryTitle.default == True)
-
-        # Add search filters
-        filter_or = []
-        if starts_with:
-            starts_with = toUnicode(starts_with.lower())
-            if starts_with in ascii_lowercase:
-                filter_or.append(LibraryTitle.simple_title.startswith(starts_with))
-            else:
-                ignore = []
-                for letter in ascii_lowercase:
-                    ignore.append(LibraryTitle.simple_title.startswith(toUnicode(letter)))
-                filter_or.append(not_(or_(*ignore)))
-
-        if search:
-            filter_or.append(LibraryTitle.simple_title.like('%%' + search + '%%'))
-
-        if len(filter_or) > 0:
-            q = q.filter(or_(*filter_or))
-
-        total_count = q.count()
-        if total_count == 0:
-            return 0, []
-
-        if order == 'release_order':
-            q = q.order_by(desc(Release.last_edit))
-        else:
-            q = q.order_by(asc(LibraryTitle.simple_title))
-
-        if limit_offset:
-            splt = splitString(limit_offset) if isinstance(limit_offset, (str, unicode)) else limit_offset
-            limit = splt[0]
-            offset = 0 if len(splt) is 1 else splt[1]
-            q = q.limit(limit).offset(offset)
-
-        # Get all movie_ids in sorted order
-        movie_ids = [m.id for m in q.all()]
-
-        # List release statuses
-        releases = db.query(Release) \
-            .filter(Release.movie_id.in_(movie_ids)) \
-            .all()
-
-        release_statuses = dict((m, set()) for m in movie_ids)
-        releases_count = dict((m, 0) for m in movie_ids)
-        for release in releases:
-            release_statuses[release.movie_id].add('%d,%d' % (release.status_id, release.quality_id))
-            releases_count[release.movie_id] += 1
-
-        # Get main movie data
-        q2 = db.query(Media) \
-            .options(joinedload_all('library.titles')) \
-            .options(joinedload_all('library.files')) \
-            .options(joinedload_all('status')) \
-            .options(joinedload_all('files'))
-
-        q2 = q2.filter(Media.id.in_(movie_ids))
-
-        results = q2.all()
-
-        # Create dict by movie id
-        movie_dict = {}
-        for movie in results:
-            movie_dict[movie.id] = movie
-
-        # List movies based on movie_ids order
-        movies = []
-        for movie_id in movie_ids:
-
-            releases = []
-            for r in release_statuses.get(movie_id):
-                x = splitString(r)
-                releases.append({'status_id': x[0], 'quality_id': x[1]})
-
-            # Merge releases with movie dict
-            movies.append(mergeDicts(movie_dict[movie_id].to_dict({
-                'library': {'titles': {}, 'files':{}},
-                'files': {},
-            }), {
-                'releases': releases,
-                'releases_count': releases_count.get(movie_id),
-            }))
-
-        db.expire_all()
-        return total_count, movies
-
-    def availableChars(self, status = None, release_status = None):
-
-        status = status or []
-        release_status = release_status or []
-
-        db = get_session()
-
-        # Make a list from string
-        if not isinstance(status, (list, tuple)):
-            status = [status]
-        if release_status and not isinstance(release_status, (list, tuple)):
-            release_status = [release_status]
-
-        q = db.query(Media)
-
-        # Filter on movie status
-        if status and len(status) > 0:
-            statuses = fireEvent('status.get', status, single = len(release_status) > 1)
-            statuses = [s.get('id') for s in statuses]
-
-            q = q.filter(Media.status_id.in_(statuses))
-
-        # Filter on release status
-        if release_status and len(release_status) > 0:
-
-            statuses = fireEvent('status.get', release_status, single = len(release_status) > 1)
-            statuses = [s.get('id') for s in statuses]
-
-            q = q.join(Media.releases) \
-                .filter(Release.status_id.in_(statuses))
-
-        q = q.join(Library, LibraryTitle) \
-            .with_entities(LibraryTitle.simple_title) \
-            .filter(LibraryTitle.default == True)
-
-        titles = q.all()
-
-        chars = set()
-        for title in titles:
-            try:
-                char = title[0][0]
-                char = char if char in ascii_lowercase else '#'
-                chars.add(str(char))
-            except:
-                log.error('Failed getting title for %s', title.libraries_id)
-
-            if len(chars) == 25:
-                break
-
-        db.expire_all()
-        return ''.join(sorted(chars))
-
-    def listView(self, **kwargs):
-
-        status = splitString(kwargs.get('status'))
-        release_status = splitString(kwargs.get('release_status'))
-        limit_offset = kwargs.get('limit_offset')
-        starts_with = kwargs.get('starts_with')
-        search = kwargs.get('search')
-        order = kwargs.get('order')
-
-        total_movies, movies = self.list(
-            status = status,
-            release_status = release_status,
-            limit_offset = limit_offset,
-            starts_with = starts_with,
-            search = search,
-            order = order
-        )
-
-        return {
-            'success': True,
-            'empty': len(movies) == 0,
-            'total': total_movies,
-            'movies': movies,
-        }
-
-    def charView(self, **kwargs):
-
-        status = splitString(kwargs.get('status', None))
-        release_status = splitString(kwargs.get('release_status', None))
-        chars = self.availableChars(status, release_status)
-
-        return {
-            'success': True,
-            'empty': len(chars) == 0,
-            'chars': chars,
-        }
 
     def add(self, params = None, force_readd = True, search_after = True, update_library = False, status_id = None):
         if not params: params = {}
@@ -421,9 +148,9 @@ class MovieBase(MovieTypeBase):
         available_status = fireEvent('status.get', 'available', single = True)
 
         ids = splitString(id)
-        for movie_id in ids:
+        for media_id in ids:
 
-            m = db.query(Media).filter_by(id = movie_id).first()
+            m = db.query(Media).filter_by(id = media_id).first()
             if not m:
                 continue
 
@@ -446,98 +173,12 @@ class MovieBase(MovieTypeBase):
 
             db.commit()
 
-            fireEvent('movie.restatus', m.id)
+            fireEvent('media.restatus', m.id)
 
             movie_dict = m.to_dict(self.default_dict)
-            fireEventAsync('movie.searcher.single', movie_dict, on_complete = self.createNotifyFront(movie_id))
+            fireEventAsync('movie.searcher.single', movie_dict, on_complete = self.createNotifyFront(media_id))
 
         db.expire_all()
         return {
             'success': True,
         }
-
-    def deleteView(self, id = '', **kwargs):
-
-        ids = splitString(id)
-        for movie_id in ids:
-            self.delete(movie_id, delete_from = kwargs.get('delete_from', 'all'))
-
-        return {
-            'success': True,
-        }
-
-    def delete(self, movie_id, delete_from = None):
-
-        db = get_session()
-
-        movie = db.query(Media).filter_by(id = movie_id).first()
-        if movie:
-            deleted = False
-            if delete_from == 'all':
-                db.delete(movie)
-                db.commit()
-                deleted = True
-            else:
-                done_status = fireEvent('status.get', 'done', single = True)
-
-                total_releases = len(movie.releases)
-                total_deleted = 0
-                new_movie_status = None
-                for release in movie.releases:
-                    if delete_from in ['wanted', 'snatched', 'late']:
-                        if release.status_id != done_status.get('id'):
-                            db.delete(release)
-                            total_deleted += 1
-                        new_movie_status = 'done'
-                    elif delete_from == 'manage':
-                        if release.status_id == done_status.get('id'):
-                            db.delete(release)
-                            total_deleted += 1
-                        new_movie_status = 'active'
-                db.commit()
-
-                if total_releases == total_deleted:
-                    db.delete(movie)
-                    db.commit()
-                    deleted = True
-                elif new_movie_status:
-                    new_status = fireEvent('status.get', new_movie_status, single = True)
-                    movie.profile_id = None
-                    movie.status_id = new_status.get('id')
-                    db.commit()
-                else:
-                    fireEvent('movie.restatus', movie.id, single = True)
-
-            if deleted:
-                fireEvent('notify.frontend', type = 'movie.deleted', data = movie.to_dict())
-
-        db.expire_all()
-        return True
-
-    def restatus(self, movie_id):
-
-        active_status, done_status = fireEvent('status.get', ['active', 'done'], single = True)
-
-        db = get_session()
-
-        m = db.query(Media).filter_by(id = movie_id).first()
-        if not m or len(m.library.titles) == 0:
-            log.debug('Can\'t restatus movie, doesn\'t seem to exist.')
-            return False
-
-        log.debug('Changing status for %s', m.library.titles[0].title)
-        if not m.profile:
-            m.status_id = done_status.get('id')
-        else:
-            move_to_wanted = True
-
-            for t in m.profile.types:
-                for release in m.releases:
-                    if t.quality.identifier is release.quality.identifier and (release.status_id is done_status.get('id') and t.finish):
-                        move_to_wanted = False
-
-            m.status_id = active_status.get('id') if move_to_wanted else done_status.get('id')
-
-        db.commit()
-
-        return True

couchpotato/core/media/movie/_base/static/list.js

@@ -52,8 +52,8 @@ var MovieList = new Class({
 
 		self.getMovies();
 
-		App.addEvent('movie.added', self.movieAdded.bind(self))
-		App.addEvent('movie.deleted', self.movieDeleted.bind(self))
+		App.on('movie.added', self.movieAdded.bind(self))
+		App.on('movie.deleted', self.movieDeleted.bind(self))
 	},
 
 	movieDeleted: function(notification){
@@ -65,6 +65,7 @@ var MovieList = new Class({
 					movie.destroy();
 					delete self.movies_added[notification.data.id];
 					self.setCounter(self.counter_count-1);
+					self.total_movies--;
 				}
 			})
 		}
@@ -75,6 +76,7 @@ var MovieList = new Class({
 	movieAdded: function(notification){
 		var self = this;
 
+		self.fireEvent('movieAdded', notification);
 		if(self.options.add_new && !self.movies_added[notification.data.id] && notification.data.status.identifier == self.options.status){
 			window.scroll(0,0);
 			self.createMovie(notification.data, 'top');
@@ -279,7 +281,7 @@ var MovieList = new Class({
 
 			// Get available chars and highlight
 			if(!available_chars && (self.navigation.isDisplayed() || self.navigation.isVisible()))
-				Api.request('movie.available_chars', {
+				Api.request('media.available_chars', {
 					'data': Object.merge({
 						'status': self.options.status
 					}, self.filter),
@@ -370,7 +372,7 @@ var MovieList = new Class({
 				'click': function(e){
 					(e).preventDefault();
 					this.set('text', 'Deleting..')
-					Api.request('movie.delete', {
+					Api.request('media.delete', {
 						'data': {
 							'id': ids.join(','),
 							'delete_from': self.options.identifier
@@ -390,6 +392,7 @@ var MovieList = new Class({
 								self.movies.erase(movie);
 								movie.destroy();
 								self.setCounter(self.counter_count-1);
+								self.total_movies--;
 							});
 
 							self.calculateSelected();
@@ -547,8 +550,9 @@ var MovieList = new Class({
 
 		}
 
-		Api.request(self.options.api_call || 'movie.list', {
+		Api.request(self.options.api_call || 'media.list', {
 			'data': Object.merge({
+				'type': 'movie',
 				'status': self.options.status,
 				'limit_offset': self.options.limit ? self.options.limit + ',' + self.offset : null
 			}, self.filter),

couchpotato/core/media/movie/_base/static/movie.actions.js

@@ -126,7 +126,9 @@ MA.Release = new Class({
 		else
 			self.showHelper();
 
-		App.addEvent('movie.searcher.ended.'+self.movie.data.id, function(notification){
+		App.on('movie.searcher.ended', function(notification){
+			if(self.movie.data.id != notification.data.id) return;
+
 			self.releases = null;
 			if(self.options_container){
 				self.options_container.destroy();
@@ -250,12 +252,14 @@ MA.Release = new Class({
 				else if(!self.next_release && status.identifier == 'available'){
 					self.next_release = release;
 				}
-				
+
 				var update_handle = function(notification) {
-					var q = self.movie.quality.getElement('.q_id' + release.quality_id), 
+					if(notification.data.id != release.id) return;
+
+					var q = self.movie.quality.getElement('.q_id' + release.quality_id),
 						status = Status.get(release.status_id),
-						new_status = Status.get(notification.data);
-					
+						new_status = Status.get(notification.data.status_id);
+
 					release.status_id = new_status.id
 					release.el.set('class', 'item ' + new_status.identifier);
 
@@ -272,7 +276,7 @@ MA.Release = new Class({
 					}
 				}
 
-				App.addEvent('release.update_status.' + release.id, update_handle); 
+				App.on('release.update_status', update_handle);
 
 			});
 
@@ -285,7 +289,7 @@ MA.Release = new Class({
 			if(self.next_release || (self.last_release && ['ignored', 'failed'].indexOf(self.last_release.status.identifier) === false)){
 
 				self.trynext_container = new Element('div.buttons.try_container').inject(self.release_container, 'top');
-				
+
 				var nr = self.next_release,
 					lr = self.last_release;
 
@@ -427,7 +431,7 @@ MA.Release = new Class({
 	markMovieDone: function(){
 		var self = this;
 
-		Api.request('movie.delete', {
+		Api.request('media.delete', {
 			'data': {
 				'id': self.movie.get('id'),
 				'delete_from': 'wanted'
@@ -446,7 +450,7 @@ MA.Release = new Class({
 
 	},
 
-	tryNextRelease: function(movie_id){
+	tryNextRelease: function(){
 		var self = this;
 
 		Api.request('movie.searcher.try_next', {
@@ -817,7 +821,7 @@ MA.Delete = new Class({
 				self.callChain();
 			},
 			function(){
-				Api.request('movie.delete', {
+				Api.request('media.delete', {
 					'data': {
 						'id': self.movie.get('id'),
 						'delete_from': self.movie.list.options.identifier

couchpotato/core/media/movie/_base/static/movie.css

@@ -1036,7 +1036,7 @@
 				text-overflow: ellipsis;
 				overflow: hidden;
 				width: 85%;
-				direction: rtl;
+				direction: ltr;
 				vertical-align: middle;
 			}
 

couchpotato/core/media/movie/_base/static/movie.js

@@ -23,23 +23,49 @@ var Movie = new Class({
 	addEvents: function(){
 		var self = this;
 
-		App.addEvent('movie.update.'+self.data.id, function(notification){
+		self.global_events = {}
+
+		// Do refresh with new data
+		self.global_events['movie.update'] = function(notification){
+			if(self.data.id != notification.data.id) return;
+
 			self.busy(false);
 			self.removeView();
 			self.update.delay(2000, self, notification);
-		});
+		}
+		App.on('movie.update', self.global_events['movie.update']);
 
+		// Add spinner on load / search
 		['movie.busy', 'movie.searcher.started'].each(function(listener){
-			App.addEvent(listener+'.'+self.data.id, function(notification){
-				if(notification.data)
+			self.global_events[listener] = function(notification){
+				if(notification.data && self.data.id == notification.data.id)
 					self.busy(true)
-			});
+			}
+			App.on(listener, self.global_events[listener]);
 		})
 
-		App.addEvent('movie.searcher.ended.'+self.data.id, function(notification){
-			if(notification.data)
+		// Remove spinner
+		self.global_events['movie.searcher.ended'] = function(notification){
+			if(notification.data && self.data.id == notification.data.id)
 				self.busy(false)
-		});
+		}
+		App.on('movie.searcher.ended', self.global_events['movie.searcher.ended']);
+
+		// Reload when releases have updated
+		self.global_events['release.update_status'] = function(notification){
+			var data = notification.data
+			if(data && self.data.id == data.movie_id){
+
+				if(!self.data.releases)
+					self.data.releases = [];
+
+				self.data.releases.push({'quality_id': data.quality_id, 'status_id': data.status_id});
+				self.updateReleases();
+			}
+		}
+
+		App.on('release.update_status', self.global_events['release.update_status']);
+
 	},
 
 	destroy: function(){
@@ -52,10 +78,9 @@ var Movie = new Class({
 		self.list.checkIfEmpty();
 
 		// Remove events
-		App.removeEvents('movie.update.'+self.data.id);
-		['movie.busy', 'movie.searcher.started'].each(function(listener){
-			App.removeEvents(listener+'.'+self.data.id);
-		})
+		Object.each(self.global_events, function(handle, listener){
+			App.off(listener, handle);
+		});
 	},
 
 	busy: function(set_busy, timeout){
@@ -179,21 +204,7 @@ var Movie = new Class({
 			});
 
 		// Add releases
-		if(self.data.releases)
-			self.data.releases.each(function(release){
-
-				var q = self.quality.getElement('.q_id'+ release.quality_id),
-					status = Status.get(release.status_id);
-
-				if(!q && (status.identifier == 'snatched' || status.identifier == 'seeding' || status.identifier == 'done'))
-					var q = self.addQuality(release.quality_id)
-
-				if (status && q && !q.hasClass(status.identifier)){
-					q.addClass(status.identifier);
-					q.set('title', (q.get('title') ? q.get('title') : '') + ' status: '+ status.label)
-				}
-
-			});
+		self.updateReleases();
 
 		Object.each(self.options.actions, function(action, key){
 			self.action[key.toLowerCase()] = action = new self.options.actions[key](self)
@@ -203,6 +214,26 @@ var Movie = new Class({
 
 	},
 
+	updateReleases: function(){
+		var self = this;
+		if(!self.data.releases || self.data.releases.length == 0) return;
+
+		self.data.releases.each(function(release){
+
+			var q = self.quality.getElement('.q_id'+ release.quality_id),
+				status = Status.get(release.status_id);
+
+			if(!q && (status.identifier == 'snatched' || status.identifier == 'seeding' || status.identifier == 'done'))
+				var q = self.addQuality(release.quality_id)
+
+			if (status && q && !q.hasClass(status.identifier)){
+				q.addClass(status.identifier);
+				q.set('title', (q.get('title') ? q.get('title') : '') + ' status: '+ status.label)
+			}
+
+		});
+	},
+
 	addQuality: function(quality_id){
 		var self = this;
 

couchpotato/core/media/movie/library/movie/main.py

@@ -151,7 +151,7 @@ class MovieLibraryPlugin(LibraryBase):
         else:
             dates = library.info.get('release_date')
 
-        if dates and dates.get('expires', 0) < time.time() or not dates:
+        if dates and (dates.get('expires', 0) < time.time() or dates.get('expires', 0) > time.time() + (604800 * 4)) or not dates:
             dates = fireEvent('movie.release_date', identifier = identifier, merge = True)
             library.info.update({'release_date': dates })
             db.commit()

couchpotato/core/media/movie/searcher/main.py

@@ -145,10 +145,10 @@ class MovieSearcher(SearcherBase, MovieTypeBase):
         default_title = getTitle(movie['library'])
         if not default_title:
             log.error('No proper info found for movie, removing it from library to cause it from having more issues.')
-            fireEvent('movie.delete', movie['id'], single = True)
+            fireEvent('media.delete', movie['id'], single = True)
             return
 
-        fireEvent('notify.frontend', type = 'movie.searcher.started.%s' % movie['id'], data = True, message = 'Searching for "%s"' % default_title)
+        fireEvent('notify.frontend', type = 'movie.searcher.started', data = {'id': movie['id']}, message = 'Searching for "%s"' % default_title)
 
 
         ret = False
@@ -192,7 +192,7 @@ class MovieSearcher(SearcherBase, MovieTypeBase):
 
             else:
                 log.info('Better quality (%s) already available or snatched for %s', (quality_type['quality']['label'], default_title))
-                fireEvent('movie.restatus', movie['id'])
+                fireEvent('media.restatus', movie['id'])
                 break
 
             # Break if CP wants to shut down
@@ -202,7 +202,7 @@ class MovieSearcher(SearcherBase, MovieTypeBase):
         if len(too_early_to_search) > 0:
             log.info2('Too early to search for %s, %s', (too_early_to_search, default_title))
 
-        fireEvent('notify.frontend', type = 'movie.searcher.ended.%s' % movie['id'], data = True)
+        fireEvent('notify.frontend', type = 'movie.searcher.ended', data = {'id': movie['id']})
 
         return ret
 
@@ -318,14 +318,14 @@ class MovieSearcher(SearcherBase, MovieTypeBase):
             'success': trynext
         }
 
-    def tryNextRelease(self, movie_id, manual = False):
+    def tryNextRelease(self, media_id, manual = False):
 
         snatched_status, done_status, ignored_status = fireEvent('status.get', ['snatched', 'done', 'ignored'], single = True)
 
         try:
             db = get_session()
             rels = db.query(Release) \
-                .filter_by(movie_id = movie_id) \
+                .filter_by(movie_id = media_id) \
                 .filter(Release.status_id.in_([snatched_status.get('id'), done_status.get('id')])) \
                 .all()
 
@@ -333,7 +333,7 @@ class MovieSearcher(SearcherBase, MovieTypeBase):
                 rel.status_id = ignored_status.get('id')
             db.commit()
 
-            movie_dict = fireEvent('movie.get', movie_id, single = True)
+            movie_dict = fireEvent('media.get', media_id = media_id, single = True)
             log.info('Trying next release for: %s', getTitle(movie_dict['library']))
             fireEvent('movie.searcher.single', movie_dict, manual = manual)
 

couchpotato/core/notifications/base.py

@@ -17,7 +17,7 @@ class Notification(Provider):
     listen_to = [
         'renamer.after', 'movie.snatched',
         'updater.available', 'updater.updated',
-        'core.message',
+        'core.message.important',
     ]
     dont_listen_to = []
 

couchpotato/core/notifications/core/main.py

@@ -21,6 +21,12 @@ class CoreNotifier(Notification):
 
     m_lock = None
 
+    listen_to = [
+        'renamer.after', 'movie.snatched',
+        'updater.available', 'updater.updated',
+        'core.message', 'core.message.important',
+    ]
+
     def __init__(self):
         super(CoreNotifier, self).__init__()
 
@@ -121,7 +127,10 @@ class CoreNotifier(Notification):
 
         for message in messages:
             if message.get('time') > last_check:
-                fireEvent('core.message', message = message.get('message'), data = message)
+                message['sticky'] = True # Always sticky core messages
+
+                message_type = 'core.message.important' if message.get('important') else 'core.message'
+                fireEvent(message_type, message = message.get('message'), data = message)
 
             if last_check < message.get('time'):
                 last_check = message.get('time')

couchpotato/core/notifications/core/static/notification.js

@@ -10,8 +10,8 @@ var NotificationBase = new Class({
 		// Listener
 		App.addEvent('unload', self.stopPoll.bind(self));
 		App.addEvent('reload', self.startInterval.bind(self, [true]));
-		App.addEvent('notification', self.notify.bind(self));
-		App.addEvent('message', self.showMessage.bind(self));
+		App.on('notification', self.notify.bind(self));
+		App.on('message', self.showMessage.bind(self));
 
 		// Add test buttons to settings page
 		App.addEvent('load', self.addTestButtons.bind(self));
@@ -50,9 +50,9 @@ var NotificationBase = new Class({
 		, 'top');
 		self.notifications.include(result);
 
-		if(result.data.important !== undefined && !result.read){
+		if((result.data.important !== undefined || result.data.sticky !== undefined) && !result.read){
 			var sticky = true
-			App.fireEvent('message', [result.message, sticky, result])
+			App.trigger('message', [result.message, sticky, result])
 		}
 		else if(!result.read){
 			self.setBadge(self.notifications.filter(function(n){ return !n.read}).length)
@@ -147,7 +147,7 @@ var NotificationBase = new Class({
 		// Process data
 		if(json){
 			Array.each(json.result, function(result){
-				App.fireEvent(result.type, result);
+				App.trigger(result.type, result);
 				if(result.message && result.read === undefined)
 					self.showMessage(result.message);
 			})

couchpotato/core/notifications/email/main.py

@@ -4,6 +4,7 @@ from couchpotato.core.logger import CPLog
 from couchpotato.core.notifications.base import Notification
 from couchpotato.environment import Env
 from email.mime.text import MIMEText
+from email.utils import formatdate, make_msgid
 import smtplib
 import traceback
 
@@ -30,6 +31,8 @@ class Email(Notification):
         message['Subject'] = self.default_title
         message['From'] = from_address
         message['To'] = to_address
+        message['Date'] = formatdate(localtime = 1)
+        message['Message-ID'] = make_msgid()
 
         try:
             # Open the SMTP connection, via SSL if requested

couchpotato/core/notifications/pushbullet/__init__.py

@@ -0,0 +1,39 @@
+from .main import Pushbullet
+
+def start():
+    return Pushbullet()
+
+config = [{
+    'name': 'pushbullet',
+    'groups': [
+        {
+            'tab': 'notifications',
+            'list': 'notification_providers',
+            'name': 'pushbullet',
+            'options': [
+                {
+                    'name': 'enabled',
+                    'default': 0,
+                    'type': 'enabler',
+                },
+                {
+                    'name': 'api_key',
+                    'label': 'User API Key'
+                },
+                {
+                    'name': 'devices',
+                    'default': '',
+                    'advanced': True,
+                    'description': 'IDs of devices to send notifications to, empty = all devices'
+                },
+                {
+                    'name': 'on_snatch',
+                    'default': 0,
+                    'type': 'bool',
+                    'advanced': True,
+                    'description': 'Also send message when movie is snatched.',
+                },
+            ],
+        }
+    ],
+}]

couchpotato/core/notifications/pushbullet/main.py

@@ -0,0 +1,86 @@
+from couchpotato.core.helpers.encoding import toUnicode
+from couchpotato.core.helpers.variable import tryInt
+from couchpotato.core.logger import CPLog
+from couchpotato.core.notifications.base import Notification
+import base64
+import json
+
+log = CPLog(__name__)
+
+
+class Pushbullet(Notification):
+
+    url = 'https://api.pushbullet.com/api/%s'
+
+    def notify(self, message = '', data = None, listener = None):
+        if not data: data = {}
+
+        devices = self.getDevices()
+        if devices is None:
+            return False
+
+        # Get all the device IDs linked to this user
+        if not len(devices):
+            response = self.request('devices')
+            if not response:
+                return False
+
+            devices += [device.get('id') for device in response['devices']]
+
+        successful = 0
+        for device in devices:
+            response = self.request(
+                'pushes',
+                cache = False,
+                device_id = device,
+                type = 'note',
+                title = self.default_title,
+                body = toUnicode(message)
+            )
+
+            if response:
+                successful += 1
+            else:
+                log.error('Unable to push notification to Pushbullet device with ID %s' % device)
+
+        return successful == len(devices)
+
+    def getDevices(self):
+        devices = [d.strip() for d in self.conf('devices').split(',')]
+
+        # Remove empty items
+        devices = [d for d in devices if len(d)]
+
+        # Break on any ids that aren't integers
+        valid_devices = []
+
+        for device_id in devices:
+            d = tryInt(device_id, None)
+
+            if not d:
+                log.error('Device ID "%s" is not valid', device_id)
+                return None
+
+            valid_devices.append(d)
+
+        return valid_devices
+
+    def request(self, method, cache = True, **kwargs):
+        try:
+            base64string = base64.encodestring('%s:' % self.conf('api_key'))[:-1]
+
+            headers = {
+                "Authorization": "Basic %s" % base64string
+            }
+
+            if cache:
+                return self.getJsonData(self.url % method, headers = headers, params = kwargs)
+            else:
+                data = self.urlopen(self.url % method, headers = headers, params = kwargs)
+                return json.loads(data)
+
+        except Exception, ex:
+            log.error('Pushbullet request failed')
+            log.debug(ex)
+
+        return None

couchpotato/core/notifications/xbmc/__init__.py

@@ -46,6 +46,14 @@ config = [{
                     'advanced': True,
                     'description': 'Only scan new movie folder at remote XBMC servers. Works if movie location is the same.',
                 },
+                {
+                    'name': 'force_full_scan',
+                    'label': 'Always do a full scan',
+                    'default': 0,
+                    'type': 'bool',
+                    'advanced': True,
+                    'description': 'Do a full scan instead of only the new movie. Useful if the XBMC path is different from the path CPS uses.',
+                },
                 {
                     'name': 'on_snatch',
                     'default': 0,

couchpotato/core/notifications/xbmc/main.py

@@ -36,7 +36,7 @@ class XBMC(Notification):
 
                 if data and data.get('destination_dir') and (not self.conf('only_first') or hosts.index(host) == 0):
                     param = {}
-                    if self.conf('remote_dir_scan') or socket.getfqdn('localhost') == socket.getfqdn(host.split(':')[0]):
+                    if not self.conf('force_full_scan') and (self.conf('remote_dir_scan') or socket.getfqdn('localhost') == socket.getfqdn(host.split(':')[0])):
                         param = {'directory': data['destination_dir']}
 
                     calls.append(('VideoLibrary.Scan', param))

couchpotato/core/plugins/automation/__init__.py

@@ -41,7 +41,7 @@ config = [{
                     'label': 'Required Genres',
                     'default': '',
                     'placeholder': 'Example: Action, Crime & Drama',
-                    'description': 'Ignore movies that don\'t contain at least one set of genres. Sets are separated by "," and each word within a set must be separated with "&"'
+                    'description': ('Ignore movies that don\'t contain at least one set of genres.', 'Sets are separated by "," and each word within a set must be separated with "&"')
                 },
                 {
                     'name': 'ignored_genres',

couchpotato/core/plugins/automation/main.py

@@ -43,7 +43,7 @@ class Automation(Plugin):
             if self.shuttingDown():
                 break
 
-            movie_dict = fireEvent('movie.get', movie_id, single = True)
+            movie_dict = fireEvent('media.get', movie_id, single = True)
             fireEvent('movie.searcher.single', movie_dict)
 
-        return True
+        return True

couchpotato/core/plugins/base.py

@@ -289,19 +289,19 @@ class Plugin(object):
         Env.get('cache').set(cache_key_md5, value, timeout)
         return value
 
-    def createNzbName(self, data, movie):
-        tag = self.cpTag(movie)
+    def createNzbName(self, data, media):
+        tag = self.cpTag(media)
         return '%s%s' % (toSafeString(toUnicode(data.get('name'))[:127 - len(tag)]), tag)
 
-    def createFileName(self, data, filedata, movie):
-        name = sp(os.path.join(self.createNzbName(data, movie)))
+    def createFileName(self, data, filedata, media):
+        name = sp(os.path.join(self.createNzbName(data, media)))
         if data.get('protocol') == 'nzb' and 'DOCTYPE nzb' not in filedata and '</nzb>' not in filedata:
             return '%s.%s' % (name, 'rar')
         return '%s.%s' % (name, data.get('protocol'))
 
-    def cpTag(self, movie):
+    def cpTag(self, media):
         if Env.setting('enabled', 'renamer'):
-            return '.cp(' + movie['library'].get('identifier') + ')' if movie['library'].get('identifier') else ''
+            return '.cp(' + media['library'].get('identifier') + ')' if media['library'].get('identifier') else ''
 
         return ''
 

couchpotato/core/plugins/manage/main.py

@@ -1,6 +1,6 @@
 from couchpotato.api import addApiView
 from couchpotato.core.event import fireEvent, addEvent, fireEventAsync
-from couchpotato.core.helpers.encoding import ss
+from couchpotato.core.helpers.encoding import sp
 from couchpotato.core.helpers.variable import splitString, getTitle
 from couchpotato.core.logger import CPLog
 from couchpotato.core.plugins.base import Plugin
@@ -79,6 +79,7 @@ class Manage(Plugin):
         try:
 
             directories = self.directories()
+            directories.sort()
             added_identifiers = []
 
             # Add some progress
@@ -111,22 +112,20 @@ class Manage(Plugin):
             if self.conf('cleanup') and full and not self.shuttingDown():
 
                 # Get movies with done status
-                total_movies, done_movies = fireEvent('movie.list', status = 'done', single = True)
+                total_movies, done_movies = fireEvent('media.list', types = 'movie', status = 'done', single = True)
 
                 for done_movie in done_movies:
                     if done_movie['library']['identifier'] not in added_identifiers:
-                        fireEvent('movie.delete', movie_id = done_movie['id'], delete_from = 'all')
+                        fireEvent('media.delete', media_id = done_movie['id'], delete_from = 'all')
                     else:
 
                         releases = fireEvent('release.for_movie', id = done_movie.get('id'), single = True)
 
                         for release in releases:
-                            if len(release.get('files', [])) == 0:
-                                fireEvent('release.delete', release['id'])
-                            else:
+                            if len(release.get('files', [])) > 0:
                                 for release_file in release.get('files', []):
                                     # Remove release not available anymore
-                                    if not os.path.isfile(ss(release_file['path'])):
+                                    if not os.path.isfile(sp(release_file['path'])):
                                         fireEvent('release.clean', release['id'])
                                         break
 
@@ -201,7 +200,7 @@ class Manage(Plugin):
 
             self.in_progress[folder]['to_go'] -= 1
             total = self.in_progress[folder]['total']
-            movie_dict = fireEvent('movie.get', identifier, single = True)
+            movie_dict = fireEvent('media.get', identifier, single = True)
 
             fireEvent('notify.frontend', type = 'movie.added', data = movie_dict, message = None if total > 5 else 'Added "%s" to manage.' % getTitle(movie_dict['library']))
 

couchpotato/core/plugins/quality/main.py

@@ -2,7 +2,7 @@ from couchpotato import get_session
 from couchpotato.api import addApiView
 from couchpotato.core.event import addEvent
 from couchpotato.core.helpers.encoding import toUnicode, ss
-from couchpotato.core.helpers.variable import mergeDicts, md5, getExt
+from couchpotato.core.helpers.variable import mergeDicts, getExt
 from couchpotato.core.logger import CPLog
 from couchpotato.core.plugins.base import Plugin
 from couchpotato.core.settings.model import Quality, Profile, ProfileType
@@ -19,14 +19,14 @@ class QualityPlugin(Plugin):
         {'identifier': 'bd50', 'hd': True, 'size': (15000, 60000), 'label': 'BR-Disk', 'alternative': ['bd25'], 'allow': ['1080p'], 'ext':[], 'tags': ['bdmv', 'certificate', ('complete', 'bluray')]},
         {'identifier': '1080p', 'hd': True, 'size': (4000, 20000), 'label': '1080p', 'width': 1920, 'height': 1080, 'alternative': [], 'allow': [], 'ext':['mkv', 'm2ts'], 'tags': ['m2ts', 'x264', 'h264']},
         {'identifier': '720p', 'hd': True, 'size': (3000, 10000), 'label': '720p', 'width': 1280, 'height': 720, 'alternative': [], 'allow': [], 'ext':['mkv', 'ts'], 'tags': ['x264', 'h264']},
-        {'identifier': 'brrip', 'hd': True, 'size': (700, 7000), 'label': 'BR-Rip', 'alternative': ['bdrip'], 'allow': ['720p', '1080p'], 'ext':['avi'], 'tags': ['hdtv', 'hdrip', 'webdl', ('web', 'dl')]},
+        {'identifier': 'brrip', 'hd': True, 'size': (700, 7000), 'label': 'BR-Rip', 'alternative': ['bdrip'], 'allow': ['720p', '1080p'], 'ext':[], 'tags': ['hdtv', 'hdrip', 'webdl', ('web', 'dl')]},
         {'identifier': 'dvdr', 'size': (3000, 10000), 'label': 'DVD-R', 'alternative': ['br2dvd'], 'allow': [], 'ext':['iso', 'img', 'vob'], 'tags': ['pal', 'ntsc', 'video_ts', 'audio_ts', ('dvd', 'r')]},
-        {'identifier': 'dvdrip', 'size': (600, 2400), 'label': 'DVD-Rip', 'width': 720, 'alternative': [], 'allow': [], 'ext':['avi', 'mpg', 'mpeg'], 'tags': [('dvd', 'rip'), ('dvd', 'xvid'), ('dvd', 'divx')]},
-        {'identifier': 'scr', 'size': (600, 1600), 'label': 'Screener', 'alternative': ['screener', 'dvdscr', 'ppvrip', 'dvdscreener', 'hdscr'], 'allow': ['dvdr', 'dvdrip', '720p'], 'ext':['avi', 'mpg', 'mpeg'], 'tags': ['webrip', ('web', 'rip')]},
-        {'identifier': 'r5', 'size': (600, 1000), 'label': 'R5', 'alternative': ['r6'], 'allow': ['dvdr'], 'ext':['avi', 'mpg', 'mpeg']},
-        {'identifier': 'tc', 'size': (600, 1000), 'label': 'TeleCine', 'alternative': ['telecine'], 'allow': [], 'ext':['avi', 'mpg', 'mpeg']},
-        {'identifier': 'ts', 'size': (600, 1000), 'label': 'TeleSync', 'alternative': ['telesync', 'hdts'], 'allow': [], 'ext':['avi', 'mpg', 'mpeg']},
-        {'identifier': 'cam', 'size': (600, 1000), 'label': 'Cam', 'alternative': ['camrip', 'hdcam'], 'allow': [], 'ext':['avi', 'mpg', 'mpeg']}
+        {'identifier': 'dvdrip', 'size': (600, 2400), 'label': 'DVD-Rip', 'width': 720, 'alternative': [], 'allow': [], 'ext':[], 'tags': [('dvd', 'rip'), ('dvd', 'xvid'), ('dvd', 'divx')]},
+        {'identifier': 'scr', 'size': (600, 1600), 'label': 'Screener', 'alternative': ['screener', 'dvdscr', 'ppvrip', 'dvdscreener', 'hdscr'], 'allow': ['dvdr', 'dvdrip', '720p', '1080p'], 'ext':[], 'tags': ['webrip', ('web', 'rip')]},
+        {'identifier': 'r5', 'size': (600, 1000), 'label': 'R5', 'alternative': ['r6'], 'allow': ['dvdr'], 'ext':[]},
+        {'identifier': 'tc', 'size': (600, 1000), 'label': 'TeleCine', 'alternative': ['telecine'], 'allow': [], 'ext':[]},
+        {'identifier': 'ts', 'size': (600, 1000), 'label': 'TeleSync', 'alternative': ['telesync', 'hdts'], 'allow': [], 'ext':[]},
+        {'identifier': 'cam', 'size': (600, 1000), 'label': 'Cam', 'alternative': ['camrip', 'hdcam'], 'allow': [], 'ext':[]}
     ]
     pre_releases = ['cam', 'ts', 'tc', 'r5', 'scr']
 
@@ -50,6 +50,8 @@ class QualityPlugin(Plugin):
 
         addEvent('app.initialize', self.fill, priority = 10)
 
+        addEvent('app.test', self.doTest)
+
     def preReleases(self):
         return self.pre_releases
 
@@ -165,9 +167,10 @@ class QualityPlugin(Plugin):
         if not extra: extra = {}
 
         # Create hash for cache
-        cache_key = md5(str([f.replace('.' + getExt(f), '') for f in files]))
+        cache_key = str([f.replace('.' + getExt(f), '') if len(getExt(f)) < 4 else f for f in files])
         cached = self.getCache(cache_key)
-        if cached and len(extra) == 0: return cached
+        if cached and len(extra) == 0:
+            return cached
 
         qualities = self.all()
 
@@ -228,11 +231,6 @@ class QualityPlugin(Plugin):
                     if len(set(words) & set(alt)) == len(alt):
                         log.debug('Found %s via %s %s in %s', (quality['identifier'], tag_type, quality.get(tag_type), cur_file))
                         score += points.get(tag_type)
-                    elif len(set(words) & set(alt)) > 0:
-                        partial = list(set(words) & set(alt))[0]
-                        if len(partial) > 2:
-                            log.debug('Found %s via partial %s %s in %s', (quality['identifier'], tag_type, quality.get(tag_type), cur_file))
-                            score += points.get(tag_type) / 3
 
                 if (isinstance(alt, (str, unicode)) and ss(alt.lower()) in cur_file.lower()):
                     log.debug('Found %s via %s %s in %s', (quality['identifier'], tag_type, quality.get(tag_type), cur_file))
@@ -285,3 +283,33 @@ class QualityPlugin(Plugin):
         if add_score != 0:
             for allow in quality.get('allow', []):
                 score[allow] -= 40 if self.cached_order[allow] < self.cached_order[quality['identifier']] else 5
+
+    def doTest(self):
+
+        tests = {
+            'Movie Name (1999)-DVD-Rip.avi': 'dvdrip',
+            'Movie Name 1999 720p Bluray.mkv': '720p',
+            'Movie Name 1999 BR-Rip 720p.avi': 'brrip',
+            'Movie Name 1999 720p Web Rip.avi': 'scr',
+            'Movie Name 1999 Web DL.avi': 'brrip',
+            'Movie.Name.1999.1080p.WEBRip.H264-Group': 'scr',
+            'Movie.Name.1999.DVDRip-Group': 'dvdrip',
+            'Movie.Name.1999.DVD-Rip-Group': 'dvdrip',
+            'Movie.Name.1999.DVD-R-Group': 'dvdr',
+        }
+
+        correct = 0
+        for name in tests:
+            success = self.guess([name]).get('identifier') == tests[name]
+            if not success:
+                log.error('%s failed check, thinks it\'s %s', (name, self.guess([name]).get('identifier')))
+
+            correct += success
+
+        if correct == len(tests):
+            log.info('Quality test successful')
+            return True
+        else:
+            log.error('Quality test failed: %s out of %s succeeded', (correct, len(tests)))
+
+

couchpotato/core/plugins/release/main.py

@@ -100,14 +100,14 @@ class Release(Plugin):
         done_status, snatched_status = fireEvent('status.get', ['done', 'snatched'], single = True)
 
         # Add movie
-        movie = db.query(Media).filter_by(library_id = group['library'].get('id')).first()
-        if not movie:
-            movie = Media(
+        media = db.query(Media).filter_by(library_id = group['library'].get('id')).first()
+        if not media:
+            media = Media(
                 library_id = group['library'].get('id'),
                 profile_id = 0,
                 status_id = done_status.get('id')
             )
-            db.add(movie)
+            db.add(media)
             db.commit()
 
         # Add Release
@@ -120,7 +120,7 @@ class Release(Plugin):
         if not rel:
             rel = Relea(
                 identifier = identifier,
-                movie = movie,
+                movie = media,
                 quality_id = group['meta_data']['quality'].get('id'),
                 status_id = done_status.get('id')
             )
@@ -142,7 +142,7 @@ class Release(Plugin):
         except:
             log.debug('Failed to attach "%s" to release: %s', (added_files, traceback.format_exc()))
 
-        fireEvent('movie.restatus', movie.id)
+        fireEvent('media.restatus', media.id)
 
         return True
 
@@ -269,7 +269,7 @@ class Release(Plugin):
                 if filedata == 'try_next':
                     return filedata
 
-            download_result = fireEvent('download', data = data, movie = media, manual = manual, filedata = filedata, single = True)
+            download_result = fireEvent('download', data = data, media = media, manual = manual, filedata = filedata, single = True)
             log.debug('Downloader result: %s', download_result)
 
             if download_result:
@@ -288,7 +288,7 @@ class Release(Plugin):
                                     value = toUnicode(download_result.get(key))
                                 )
                                 rls.info.append(rls_info)
-                        db.commit()
+                            db.commit()
 
                         log_movie = '%s (%s) in %s' % (getTitle(media['library']), media['library']['year'], rls.quality.label)
                         snatch_message = 'Snatched "%s": %s' % (data.get('name'), log_movie)
@@ -446,6 +446,6 @@ class Release(Plugin):
             db.commit()
 
             #Update all movie info as there is no release update function
-            fireEvent('notify.frontend', type = 'release.update_status.%s' % rel.id, data = status.get('id'))
+            fireEvent('notify.frontend', type = 'release.update_status', data = rel.to_dict())
 
         return True

couchpotato/core/plugins/renamer/__init__.py

@@ -93,7 +93,7 @@ config = [{
                     'default': 1,
                     'type': 'int',
                     'unit': 'min(s)',
-                    'description': 'Detect movie status every X minutes. Will start the renamer if movie is <strong>completed</strong> or handle <strong>failed</strong> download if these options are enabled',
+                    'description': ('Detect movie status every X minutes.', 'Will start the renamer if movie is <strong>completed</strong> or handle <strong>failed</strong> download if these options are enabled'),
                 },
                 {
                     'advanced': True,
@@ -122,13 +122,13 @@ config = [{
                     'advanced': True,
                     'name': 'separator',
                     'label': 'File-Separator',
-                    'description': 'Replace all the spaces with a character. Example: ".", "-" (without quotes). Leave empty to use spaces.',
+                    'description': ('Replace all the spaces with a character.', 'Example: ".", "-" (without quotes). Leave empty to use spaces.'),
                 },
                 {
                     'advanced': True,
                     'name': 'foldersep',
                     'label': 'Folder-Separator',
-                    'description': 'Replace all the spaces with a character. Example: ".", "-" (without quotes). Leave empty to use spaces.',
+                    'description': ('Replace all the spaces with a character.', 'Example: ".", "-" (without quotes). Leave empty to use spaces.'),
                 },
                 {
                     'name': 'file_action',
@@ -136,7 +136,7 @@ config = [{
                     'default': 'link',
                     'type': 'dropdown',
                     'values': [('Link', 'link'), ('Copy', 'copy'), ('Move', 'move')],
-                    'description': '<strong>Link</strong> or <strong>Copy</strong> after downloading completed (and allow for seeding), or <strong>Move</strong> after seeding completed. Link first tries <a href="http://en.wikipedia.org/wiki/Hard_link">hard link</a>, then <a href="http://en.wikipedia.org/wiki/Sym_link">sym link</a> and falls back to Copy.',
+                    'description': ('<strong>Link</strong>, <strong>Copy</strong> or <strong>Move</strong> after download completed.', 'Link first tries <a href="http://en.wikipedia.org/wiki/Hard_link">hard link</a>, then <a href="http://en.wikipedia.org/wiki/Sym_link">sym link</a> and falls back to Copy. It is perfered to use link when downloading torrents as it will save you space, while still beeing able to seed.'),
                     'advanced': True,
                 },
                 {

couchpotato/core/plugins/renamer/main.py

@@ -3,7 +3,7 @@ from couchpotato.api import addApiView
 from couchpotato.core.event import addEvent, fireEvent, fireEventAsync
 from couchpotato.core.helpers.encoding import toUnicode, ss, sp
 from couchpotato.core.helpers.variable import getExt, mergeDicts, getTitle, \
-    getImdb, link, symlink, tryInt, splitString
+    getImdb, link, symlink, tryInt, splitString, fnEscape
 from couchpotato.core.logger import CPLog
 from couchpotato.core.plugins.base import Plugin
 from couchpotato.core.settings.model import Library, File, Profile, Release, \
@@ -30,10 +30,10 @@ class Renamer(Plugin):
             'desc': 'For the renamer to check for new files to rename in a folder',
             'params': {
                 'async': {'desc': 'Optional: Set to 1 if you dont want to fire the renamer.scan asynchronous.'},
-                'base_folder': {'desc': 'Optional: The folder to find releases in. Leave empty for default folder.'},
-                'movie_folder': {'desc': 'Optional: The folder of a specific release to scan. Don\'t use in combination with base_folder.'},
+                'media_folder': {'desc': 'Optional: The folder of the media to scan. Keep empty for default renamer folder.'},
+                'files': {'desc': 'Optional: Provide the release files if more releases are in the same media_folder, delimited with a \'|\'. Note that no dedicated release folder is expected for releases with one file.'},
                 'downloader' : {'desc': 'Optional: The downloader the release has been downloaded with. \'download_id\' is required with this option.'},
-                'download_id': {'desc': 'Optional: The nzb/torrent ID of the release in movie_folder. \'downloader\' is required with this option.'},
+                'download_id': {'desc': 'Optional: The nzb/torrent ID of the release in media_folder. \'downloader\' is required with this option.'},
                 'status': {'desc': 'Optional: The status of the release: \'completed\' (default) or \'seeding\''},
             },
         })
@@ -64,26 +64,31 @@ class Renamer(Plugin):
     def scanView(self, **kwargs):
 
         async = tryInt(kwargs.get('async', 0))
-        movie_folder = sp(kwargs.get('movie_folder'))
-        base_folder = sp(kwargs.get('base_folder'))
+        media_folder = sp(kwargs.get('media_folder'))
+
+        # Backwards compatibility, to be removed after a few versions :)
+        if not media_folder:
+            media_folder = sp(kwargs.get('movie_folder'))
+
         downloader = kwargs.get('downloader')
         download_id = kwargs.get('download_id')
+        files = '|'.join([sp(filename) for filename in splitString(kwargs.get('files'), '|')])
         status = kwargs.get('status', 'completed')
 
-        release_download = None
-        if not base_folder and movie_folder:
-            release_download = {'folder': movie_folder}
-            release_download.update({'id': download_id, 'downloader': downloader, 'status': status} if download_id else {})
+        release_download = {'folder': media_folder} if media_folder else None
+        if release_download:
+            release_download.update({'id': download_id, 'downloader': downloader, 'status': status, 'files': files} if download_id else {})
 
         fire_handle = fireEvent if not async else fireEventAsync
 
-        fire_handle('renamer.scan', base_folder = base_folder, release_download = release_download)
+        fire_handle('renamer.scan', release_download)
 
         return {
             'success': True
         }
 
-    def scan(self, base_folder = None, release_download = None):
+    def scan(self, release_download = None):
+        if not release_download: release_download = {}
 
         if self.isDisabled():
             return
@@ -92,11 +97,11 @@ class Renamer(Plugin):
             log.info('Renamer is already running, if you see this often, check the logs above for errors.')
             return
 
-        from_folder = sp(self.conf('from') if not base_folder else base_folder)
+        from_folder = sp(self.conf('from'))
         to_folder = sp(self.conf('to'))
 
-        # Get movie folder to process
-        movie_folder = sp(release_download and release_download.get('folder'))
+        # Get media folder to process
+        media_folder = release_download.get('folder')
 
         # Get all folders that should not be processed
         no_process = [to_folder]
@@ -104,7 +109,7 @@ class Renamer(Plugin):
         no_process.extend([item['destination'] for item in cat_list])
         try:
             if Env.setting('library', section = 'manage').strip():
-                no_process.extend(splitString(Env.setting('library', section = 'manage'), '::'))
+                no_process.extend([sp(manage_folder) for manage_folder in splitString(Env.setting('library', section = 'manage'), '::')])
         except:
             pass
 
@@ -114,40 +119,40 @@ class Renamer(Plugin):
             return
         else:
             for item in no_process:
-                if from_folder in item:
-                    log.error('To protect your data, the movie libraries can\'t be inside of or the same as the "from" folder.')
+                if '%s%s' % (from_folder, os.path.sep) in item:
+                    log.error('To protect your data, the media libraries can\'t be inside of or the same as the "from" folder.')
                     return
 
-        # Check to see if the no_process folders are inside the provided movie_folder
-        if movie_folder and not os.path.isdir(movie_folder):
-            log.debug('The provided movie folder %s does not exist. Trying to find it in the \'from\' folder.', movie_folder)
+        # Check to see if the no_process folders are inside the provided media_folder
+        if media_folder and not os.path.isdir(media_folder):
+            log.debug('The provided media folder %s does not exist. Trying to find it in the \'from\' folder.', media_folder)
 
             # Update to the from folder
-            if len(release_download.get('files')) == 1:
-                new_movie_folder = from_folder
+            if len(splitString(release_download.get('files'), '|')) == 1:
+                new_media_folder = from_folder
             else:
-                new_movie_folder = sp(os.path.join(from_folder, os.path.basename(movie_folder)))
+                new_media_folder = os.path.join(from_folder, os.path.basename(media_folder))
 
-            if not os.path.isdir(new_movie_folder):
-                log.error('The provided movie folder %s does not exist and could also not be found in the \'from\' folder.', movie_folder)
+            if not os.path.isdir(new_media_folder):
+                log.error('The provided media folder %s does not exist and could also not be found in the \'from\' folder.', media_folder)
                 return
 
             # Update the files
-            new_files = [os.path.join(new_movie_folder, os.path.relpath(filename, movie_folder)) for filename in splitString(release_download.get('files'), '|')]
+            new_files = [os.path.join(new_media_folder, os.path.relpath(filename, media_folder)) for filename in splitString(release_download.get('files'), '|')]
             if new_files and not os.path.isfile(new_files[0]):
-                log.error('The provided movie folder %s does not exist and its files could also not be found in the \'from\' folder.', movie_folder)
+                log.error('The provided media folder %s does not exist and its files could also not be found in the \'from\' folder.', media_folder)
                 return
 
             # Update release_download info to the from folder
-            log.debug('Release %s found in the \'from\' folder.', movie_folder)
-            release_download['folder'] = new_movie_folder
+            log.debug('Release %s found in the \'from\' folder.', media_folder)
+            release_download['folder'] = new_media_folder
             release_download['files'] = '|'.join(new_files)
-            movie_folder = new_movie_folder
+            media_folder = new_media_folder
 
-        if movie_folder:
+        if media_folder:
             for item in no_process:
-                if movie_folder in item:
-                    log.error('To protect your data, the movie libraries can\'t be inside of or the same as the provided movie folder.')
+                if '%s%s' % (media_folder, os.path.sep) in item:
+                    log.error('To protect your data, the media libraries can\'t be inside of or the same as the provided media folder.')
                     return
 
         # Make sure a checkSnatched marked all downloads/seeds as such
@@ -156,26 +161,26 @@ class Renamer(Plugin):
 
         self.renaming_started = True
 
-        # make sure the movie folder name is included in the search
+        # make sure the media folder name is included in the search
         folder = None
         files = []
-        if movie_folder:
-            log.info('Scanning movie folder %s...', movie_folder)
-            folder = os.path.dirname(movie_folder)
+        if media_folder:
+            log.info('Scanning media folder %s...', media_folder)
+            folder = os.path.dirname(media_folder)
 
             if release_download.get('files', ''):
                 files = splitString(release_download['files'], '|')
 
                 # If there is only one file in the torrent, the downloader did not create a subfolder
                 if len(files) == 1:
-                    folder = movie_folder
+                    folder = media_folder
             else:
                 # Get all files from the specified folder
                 try:
-                    for root, folders, names in os.walk(movie_folder):
-                        files.extend([os.path.join(root, name) for name in names])
+                    for root, folders, names in os.walk(media_folder):
+                        files.extend([sp(os.path.join(root, name)) for name in names])
                 except:
-                    log.error('Failed getting files from %s: %s', (movie_folder, traceback.format_exc()))
+                    log.error('Failed getting files from %s: %s', (media_folder, traceback.format_exc()))
 
         db = get_session()
 
@@ -185,7 +190,7 @@ class Renamer(Plugin):
         # Unpack any archives
         extr_files = None
         if self.conf('unrar'):
-            folder, movie_folder, files, extr_files = self.extractFiles(folder = folder, movie_folder = movie_folder, files = files,
+            folder, media_folder, files, extr_files = self.extractFiles(folder = folder, media_folder = media_folder, files = files,
                                                                         cleanup = self.conf('cleanup') and not self.downloadIsTorrent(release_download))
 
         groups = fireEvent('scanner.scan', folder = folder if folder else from_folder,
@@ -496,8 +501,8 @@ class Renamer(Plugin):
                     if os.path.isfile(src):
                         os.remove(src)
 
-                        parent_dir = sp(os.path.dirname(src))
-                        if delete_folders.count(parent_dir) == 0 and os.path.isdir(parent_dir) and not parent_dir in [destination, movie_folder] and not from_folder in parent_dir:
+                        parent_dir = os.path.dirname(src)
+                        if delete_folders.count(parent_dir) == 0 and os.path.isdir(parent_dir) and not '%s%s' % (parent_dir, os.path.sep) in [destination, media_folder] and not '%s%s' % (from_folder, os.path.sep) in parent_dir:
                             delete_folders.append(parent_dir)
 
                 except:
@@ -529,7 +534,7 @@ class Renamer(Plugin):
                         self.tagRelease(group = group, tag = 'failed_rename')
 
             # Tag folder if it is in the 'from' folder and it will not be removed because it is a torrent
-            if self.movieInFromFolder(movie_folder) and self.downloadIsTorrent(release_download):
+            if self.movieInFromFolder(media_folder) and self.downloadIsTorrent(release_download):
                 self.tagRelease(group = group, tag = 'renamed_already')
 
             # Remove matching releases
@@ -541,12 +546,12 @@ class Renamer(Plugin):
                     log.error('Failed removing %s: %s', (release.identifier, traceback.format_exc()))
 
             if group['dirname'] and group['parentdir'] and not self.downloadIsTorrent(release_download):
-                if movie_folder:
+                if media_folder:
                     # Delete the movie folder
-                    group_folder = movie_folder
+                    group_folder = media_folder
                 else:
                     # Delete the first empty subfolder in the tree relative to the 'from' folder
-                    group_folder = sp(os.path.join(from_folder, os.path.relpath(group['parentdir'], from_folder)).split(os.path.sep)[0])
+                    group_folder = sp(os.path.join(from_folder, os.path.relpath(group['parentdir'], from_folder).split(os.path.sep)[0]))
 
                 try:
                     log.info('Deleting folder: %s', group_folder)
@@ -606,7 +611,7 @@ Remove it if you want it to be renamed (again, or at least let it try again)
         elif isinstance(release_download, dict):
             # Tag download_files if they are known
             if release_download['files']:
-                tag_files = release_download['files'].split('|')
+                tag_files = splitString(release_download['files'], '|')
 
             # Tag all files in release folder
             else:
@@ -614,6 +619,11 @@ Remove it if you want it to be renamed (again, or at least let it try again)
                     tag_files.extend([os.path.join(root, name) for name in names])
 
         for filename in tag_files:
+
+            # Dont tag .ignore files
+            if os.path.splitext(filename)[1] == '.ignore':
+                continue
+
             tag_filename = '%s.%s.ignore' % (os.path.splitext(filename)[0], tag)
             if not os.path.isfile(tag_filename):
                 self.createFile(tag_filename, text)
@@ -630,21 +640,21 @@ Remove it if you want it to be renamed (again, or at least let it try again)
 
         # Untag download_files if they are known
         if release_download['files']:
-            tag_files = release_download['files'].split('|')
+            tag_files = splitString(release_download['files'], '|')
 
         # Untag all files in release folder
         else:
             for root, folders, names in os.walk(release_download['folder']):
-                tag_files.extend([os.path.join(root, name) for name in names if not os.path.splitext(name)[1] == '.ignore'])
+                tag_files.extend([sp(os.path.join(root, name)) for name in names if not os.path.splitext(name)[1] == '.ignore'])
 
         # Find all .ignore files in folder
         ignore_files = []
         for root, dirnames, filenames in os.walk(folder):
-            ignore_files.extend(fnmatch.filter([os.path.join(root, filename) for filename in filenames], '*%s.ignore' % tag))
+            ignore_files.extend(fnmatch.filter([sp(os.path.join(root, filename)) for filename in filenames], '*%s.ignore' % tag))
 
         # Match all found ignore files with the tag_files and delete if found
         for tag_file in tag_files:
-            ignore_file = fnmatch.filter(ignore_files, '%s.%s.ignore' % (re.escape(os.path.splitext(tag_file)[0]), tag if tag else '*'))
+            ignore_file = fnmatch.filter(ignore_files, fnEscape('%s.%s.ignore' % (os.path.splitext(tag_file)[0], tag if tag else '*')))
             for filename in ignore_file:
                 try:
                     os.remove(filename)
@@ -664,20 +674,20 @@ Remove it if you want it to be renamed (again, or at least let it try again)
 
         # Find tag on download_files if they are known
         if release_download['files']:
-            tag_files = release_download['files'].split('|')
+            tag_files = splitString(release_download['files'], '|')
 
         # Find tag on all files in release folder
         else:
             for root, folders, names in os.walk(release_download['folder']):
-                tag_files.extend([os.path.join(root, name) for name in names if not os.path.splitext(name)[1] == '.ignore'])
+                tag_files.extend([sp(os.path.join(root, name)) for name in names if not os.path.splitext(name)[1] == '.ignore'])
 
         # Find all .ignore files in folder
         for root, dirnames, filenames in os.walk(folder):
-            ignore_files.extend(fnmatch.filter([os.path.join(root, filename) for filename in filenames], '*%s.ignore' % tag))
+            ignore_files.extend(fnmatch.filter([sp(os.path.join(root, filename)) for filename in filenames], '*%s.ignore' % tag))
 
         # Match all found ignore files with the tag_files and return True found
         for tag_file in tag_files:
-            ignore_file = fnmatch.filter(ignore_files, '%s.%s.ignore' % (os.path.splitext(tag_file)[0], tag if tag else '*'))
+            ignore_file = fnmatch.filter(ignore_files, fnEscape('%s.%s.ignore' % (os.path.splitext(tag_file)[0], tag if tag else '*')))
             if ignore_file:
                 return True
 
@@ -772,7 +782,7 @@ Remove it if you want it to be renamed (again, or at least let it try again)
         return string
 
     def deleteEmptyFolder(self, folder, show_error = True):
-        folder = ss(folder)
+        folder = sp(folder)
 
         loge = log.error if show_error else log.debug
         for root, dirs, files in os.walk(folder):
@@ -806,126 +816,159 @@ Remove it if you want it to be renamed (again, or at least let it try again)
             Release.status_id.in_([snatched_status.get('id'), seeding_status.get('id'), missing_status.get('id')])
         ).all()
 
+        if not rels:
+            #No releases found that need status checking
+            self.checking_snatched = False
+            return True
+
+        # Collect all download information with the download IDs from the releases
+        download_ids = []
+        try:
+            for rel in rels:
+                rel_dict = rel.to_dict({'info': {}})
+                if rel_dict['info'].get('download_id') and rel_dict['info'].get('download_downloader'):
+                    download_ids.append({'id': rel_dict['info']['download_id'], 'downloader': rel_dict['info']['download_downloader']})
+        except:
+            log.error('Error getting download IDs from database')
+            self.checking_snatched = False
+            return False
+
+        release_downloads = fireEvent('download.status', download_ids, merge = True)
+        if not release_downloads:
+            log.debug('Download status functionality is not implemented for any active downloaders.')
+            fireEvent('renamer.scan')
+
+            self.checking_snatched = False
+            return True
+
         scan_releases = []
         scan_required = False
 
-        if rels:
-            log.debug('Checking status snatched releases...')
+        log.debug('Checking status snatched releases...')
 
-            release_downloads = fireEvent('download.status', merge = True)
-            if not release_downloads:
-                log.debug('Download status functionality is not implemented for active downloaders.')
-                scan_required = True
-            else:
-                try:
-                    for rel in rels:
-                        rel_dict = rel.to_dict({'info': {}})
-                        movie_dict = fireEvent('movie.get', rel.movie_id, single = True)
+        try:
+            for rel in rels:
+                rel_dict = rel.to_dict({'info': {}})
+                movie_dict = fireEvent('media.get', media_id = rel.movie_id, single = True)
 
-                        if not isinstance(rel_dict['info'], (dict)):
-                            log.error('Faulty release found without any info, ignoring.')
+                if not isinstance(rel_dict['info'], (dict)):
+                    log.error('Faulty release found without any info, ignoring.')
+                    fireEvent('release.update_status', rel.id, status = ignored_status, single = True)
+                    continue
+
+                # Check if download ID is available
+                if not rel_dict['info'].get('download_id') or not rel_dict['info'].get('download_downloader'):
+                    log.debug('Download status functionality is not implemented for downloader (%s) of release %s.', (rel_dict['info'].get('download_downloader', 'unknown'), rel_dict['info']['name']))
+                    scan_required = True
+
+                    # Continue with next release
+                    continue
+
+                # Find release in downloaders
+                nzbname = self.createNzbName(rel_dict['info'], movie_dict)
+
+                for release_download in release_downloads:
+                    found_release = False
+                    if rel_dict['info'].get('download_id'):
+                        if release_download['id'] == rel_dict['info']['download_id'] and release_download['downloader'] == rel_dict['info']['download_downloader']:
+                            log.debug('Found release by id: %s', release_download['id'])
+                            found_release = True
+                            break
+                    else:
+                        if release_download['name'] == nzbname or rel_dict['info']['name'] in release_download['name'] or getImdb(release_download['name']) == movie_dict['library']['identifier']:
+                            log.debug('Found release by release name or imdb ID: %s', release_download['name'])
+                            found_release = True
+                            break
+
+                if not found_release:
+                    log.info('%s not found in downloaders', nzbname)
+
+                    #Check status if already missing and for how long, if > 1 week, set to ignored else to missing
+                    if rel.status_id == missing_status.get('id'):
+                        if rel.last_edit < int(time.time()) - 7 * 24 * 60 * 60:
                             fireEvent('release.update_status', rel.id, status = ignored_status, single = True)
-                            continue
+                    else:
+                        # Set the release to missing
+                        fireEvent('release.update_status', rel.id, status = missing_status, single = True)
 
-                        # check status
-                        nzbname = self.createNzbName(rel_dict['info'], movie_dict)
+                    # Continue with next release
+                    continue
 
-                        found = False
-                        for release_download in release_downloads:
-                            found_release = False
-                            if rel_dict['info'].get('download_id'):
-                                if release_download['id'] == rel_dict['info']['download_id'] and release_download['downloader'] == rel_dict['info']['download_downloader']:
-                                    log.debug('Found release by id: %s', release_download['id'])
-                                    found_release = True
+                # Log that we found the release
+                timeleft = 'N/A' if release_download['timeleft'] == -1 else release_download['timeleft']
+                log.debug('Found %s: %s, time to go: %s', (release_download['name'], release_download['status'].upper(), timeleft))
+
+                # Check status of release
+                if release_download['status'] == 'busy':
+                    # Set the release to snatched if it was missing before
+                    fireEvent('release.update_status', rel.id, status = snatched_status, single = True)
+
+                    # Tag folder if it is in the 'from' folder and it will not be processed because it is still downloading
+                    if self.movieInFromFolder(release_download['folder']):
+                        self.tagRelease(release_download = release_download, tag = 'downloading')
+
+                elif release_download['status'] == 'seeding':
+                    #If linking setting is enabled, process release
+                    if self.conf('file_action') != 'move' and not rel.status_id == seeding_status.get('id') and self.statusInfoComplete(release_download):
+                        log.info('Download of %s completed! It is now being processed while leaving the original files alone for seeding. Current ratio: %s.', (release_download['name'], release_download['seed_ratio']))
+
+                        # Remove the downloading tag
+                        self.untagRelease(release_download = release_download, tag = 'downloading')
+
+                        # Scan and set the torrent to paused if required
+                        release_download.update({'pause': True, 'scan': True, 'process_complete': False})
+                        scan_releases.append(release_download)
+                    else:
+                        #let it seed
+                        log.debug('%s is seeding with ratio: %s', (release_download['name'], release_download['seed_ratio']))
+
+                        # Set the release to seeding
+                        fireEvent('release.update_status', rel.id, status = seeding_status, single = True)
+
+                elif release_download['status'] == 'failed':
+                    # Set the release to failed
+                    fireEvent('release.update_status', rel.id, status = failed_status, single = True)
+
+                    fireEvent('download.remove_failed', release_download, single = True)
+
+                    if self.conf('next_on_failed'):
+                        fireEvent('movie.searcher.try_next_release', media_id = rel.movie_id)
+
+                elif release_download['status'] == 'completed':
+                    log.info('Download of %s completed!', release_download['name'])
+
+                    #Make sure the downloader sent over a path to look in
+                    if self.statusInfoComplete(release_download):
+
+                        # If the release has been seeding, process now the seeding is done
+                        if rel.status_id == seeding_status.get('id'):
+                            if self.conf('file_action') != 'move':
+                                # Set the release to done as the movie has already been renamed
+                                fireEvent('release.update_status', rel.id, status = downloaded_status, single = True)
+
+                                # Allow the downloader to clean-up
+                                release_download.update({'pause': False, 'scan': False, 'process_complete': True})
+                                scan_releases.append(release_download)
                             else:
-                                if release_download['name'] == nzbname or rel_dict['info']['name'] in release_download['name'] or getImdb(release_download['name']) == movie_dict['library']['identifier']:
-                                    found_release = True
+                                # Scan and Allow the downloader to clean-up
+                                release_download.update({'pause': False, 'scan': True, 'process_complete': True})
+                                scan_releases.append(release_download)
 
-                            if found_release:
-                                timeleft = 'N/A' if release_download['timeleft'] == -1 else release_download['timeleft']
-                                log.debug('Found %s: %s, time to go: %s', (release_download['name'], release_download['status'].upper(), timeleft))
+                        else:
+                            # Set the release to snatched if it was missing before
+                            fireEvent('release.update_status', rel.id, status = snatched_status, single = True)
 
-                                if release_download['status'] == 'busy':
-                                    # Set the release to snatched if it was missing before
-                                    fireEvent('release.update_status', rel.id, status = snatched_status, single = True)
+                            # Remove the downloading tag
+                            self.untagRelease(release_download = release_download, tag = 'downloading')
 
-                                    # Tag folder if it is in the 'from' folder and it will not be processed because it is still downloading
-                                    if self.movieInFromFolder(release_download['folder']):
-                                        self.tagRelease(release_download = release_download, tag = 'downloading')
+                            # Scan and Allow the downloader to clean-up
+                            release_download.update({'pause': False, 'scan': True, 'process_complete': True})
+                            scan_releases.append(release_download)
+                    else:
+                        scan_required = True
 
-                                elif release_download['status'] == 'seeding':
-                                    #If linking setting is enabled, process release
-                                    if self.conf('file_action') != 'move' and not rel.status_id == seeding_status.get('id') and self.statusInfoComplete(release_download):
-                                        log.info('Download of %s completed! It is now being processed while leaving the original files alone for seeding. Current ratio: %s.', (release_download['name'], release_download['seed_ratio']))
-
-                                        # Remove the downloading tag
-                                        self.untagRelease(release_download = release_download, tag = 'downloading')
-
-                                        # Scan and set the torrent to paused if required
-                                        release_download.update({'pause': True, 'scan': True, 'process_complete': False})
-                                        scan_releases.append(release_download)
-                                    else:
-                                        #let it seed
-                                        log.debug('%s is seeding with ratio: %s', (release_download['name'], release_download['seed_ratio']))
-
-                                        # Set the release to seeding
-                                        fireEvent('release.update_status', rel.id, status = seeding_status, single = True)
-
-                                elif release_download['status'] == 'failed':
-                                    # Set the release to failed
-                                    fireEvent('release.update_status', rel.id, status = failed_status, single = True)
-
-                                    fireEvent('download.remove_failed', release_download, single = True)
-
-                                    if self.conf('next_on_failed'):
-                                        fireEvent('movie.searcher.try_next_release', movie_id = rel.movie_id)
-                                elif release_download['status'] == 'completed':
-                                    log.info('Download of %s completed!', release_download['name'])
-                                    if self.statusInfoComplete(release_download):
-
-                                        # If the release has been seeding, process now the seeding is done
-                                        if rel.status_id == seeding_status.get('id'):
-                                            if self.conf('file_action') != 'move':
-                                                # Set the release to done as the movie has already been renamed
-                                                fireEvent('release.update_status', rel.id, status = downloaded_status, single = True)
-
-                                                # Allow the downloader to clean-up
-                                                release_download.update({'pause': False, 'scan': False, 'process_complete': True})
-                                                scan_releases.append(release_download)
-                                            else:
-                                                # Scan and Allow the downloader to clean-up
-                                                release_download.update({'pause': False, 'scan': True, 'process_complete': True})
-                                                scan_releases.append(release_download)
-
-                                        else:
-                                            # Set the release to snatched if it was missing before
-                                            fireEvent('release.update_status', rel.id, status = snatched_status, single = True)
-
-                                            # Remove the downloading tag
-                                            self.untagRelease(release_download = release_download, tag = 'downloading')
-
-                                            # Scan and Allow the downloader to clean-up
-                                            release_download.update({'pause': False, 'scan': True, 'process_complete': True})
-                                            scan_releases.append(release_download)
-                                    else:
-                                        scan_required = True
-
-                                found = True
-                                break
-
-                        if not found:
-                            log.info('%s not found in downloaders', nzbname)
-
-                            #Check status if already missing and for how long, if > 1 week, set to ignored else to missing
-                            if rel.status_id == missing_status.get('id'):
-                                if rel.last_edit < int(time.time()) - 7 * 24 * 60 * 60:
-                                    fireEvent('release.update_status', rel.id, status = ignored_status, single = True)
-                            else:
-                                # Set the release to missing
-                                fireEvent('release.update_status', rel.id, status = missing_status, single = True)
-
-                except:
-                    log.error('Failed checking for release in downloader: %s', traceback.format_exc())
+        except:
+            log.error('Failed checking for release in downloader: %s', traceback.format_exc())
 
         # The following can either be done here, or inside the scanner if we pass it scan_items in one go
         for release_download in scan_releases:
@@ -948,7 +991,6 @@ Remove it if you want it to be renamed (again, or at least let it try again)
             fireEvent('renamer.scan')
 
         self.checking_snatched = False
-
         return True
 
     def extendReleaseDownload(self, release_download):
@@ -995,10 +1037,10 @@ Remove it if you want it to be renamed (again, or at least let it try again)
     def statusInfoComplete(self, release_download):
         return release_download['id'] and release_download['downloader'] and release_download['folder']
 
-    def movieInFromFolder(self, movie_folder):
-        return (movie_folder and sp(self.conf('from')) in movie_folder) or not movie_folder
+    def movieInFromFolder(self, media_folder):
+        return media_folder and '%s%s' % (sp(self.conf('from')), os.path.sep) in sp(media_folder) or not media_folder
 
-    def extractFiles(self, folder = None, movie_folder = None, files = None, cleanup = False):
+    def extractFiles(self, folder = None, media_folder = None, files = None, cleanup = False):
         if not files: files = []
 
         # RegEx for finding rar files
@@ -1013,12 +1055,12 @@ Remove it if you want it to be renamed (again, or at least let it try again)
             folder = from_folder
 
         check_file_date = True
-        if movie_folder:
+        if media_folder:
             check_file_date = False
 
         if not files:
             for root, folders, names in os.walk(folder):
-                files.extend([os.path.join(root, name) for name in names])
+                files.extend([sp(os.path.join(root, name)) for name in names])
 
         # Find all archive files
         archives = [re.search(archive_regex, name).groupdict() for name in files if re.search(archive_regex, name)]
@@ -1064,13 +1106,13 @@ Remove it if you want it to be renamed (again, or at least let it try again)
             log.info('Archive %s found. Extracting...', os.path.basename(archive['file']))
             try:
                 rar_handle = RarFile(archive['file'])
-                extr_path = sp(os.path.join(from_folder, os.path.relpath(os.path.dirname(archive['file']), folder)))
+                extr_path = os.path.join(from_folder, os.path.relpath(os.path.dirname(archive['file']), folder))
                 self.makeDir(extr_path)
                 for packedinfo in rar_handle.infolist():
-                    if not packedinfo.isdir and not os.path.isfile(os.path.join(extr_path, os.path.basename(packedinfo.filename))):
+                    if not packedinfo.isdir and not os.path.isfile(sp(os.path.join(extr_path, os.path.basename(packedinfo.filename)))):
                         log.debug('Extracting %s...', packedinfo.filename)
                         rar_handle.extract(condition = [packedinfo.index], path = extr_path, withSubpath = False, overwrite = False)
-                        extr_files.append(os.path.join(extr_path, os.path.basename(packedinfo.filename)))
+                        extr_files.append(sp(os.path.join(extr_path, os.path.basename(packedinfo.filename))))
                 del rar_handle
             except Exception, e:
                 log.error('Failed to extract %s: %s %s', (archive['file'], e, traceback.format_exc()))
@@ -1087,9 +1129,9 @@ Remove it if you want it to be renamed (again, or at least let it try again)
                 files.remove(filename)
 
         # Move the rest of the files and folders if any files are extracted to the from folder (only if folder was provided)
-        if extr_files and sp(folder) != from_folder:
+        if extr_files and folder != from_folder:
             for leftoverfile in list(files):
-                move_to = sp(os.path.join(from_folder, os.path.relpath(leftoverfile, folder)))
+                move_to = os.path.join(from_folder, os.path.relpath(leftoverfile, folder))
 
                 try:
                     self.makeDir(os.path.dirname(move_to))
@@ -1109,18 +1151,18 @@ Remove it if you want it to be renamed (again, or at least let it try again)
 
             if cleanup:
                 # Remove all left over folders
-                log.debug('Removing old movie folder %s...', movie_folder)
-                self.deleteEmptyFolder(movie_folder)
+                log.debug('Removing old movie folder %s...', media_folder)
+                self.deleteEmptyFolder(media_folder)
 
-            movie_folder = sp(os.path.join(from_folder, os.path.relpath(movie_folder, folder)))
+            media_folder = os.path.join(from_folder, os.path.relpath(media_folder, folder))
             folder = from_folder
 
         if extr_files:
             files.extend(extr_files)
 
-        # Cleanup files and folder if movie_folder was not provided
-        if not movie_folder:
+        # Cleanup files and folder if media_folder was not provided
+        if not media_folder:
             files = []
             folder = None
 
-        return folder, movie_folder, files, extr_files
+        return folder, media_folder, files, extr_files

couchpotato/core/plugins/scanner/main.py

@@ -124,7 +124,7 @@ class Scanner(Plugin):
             try:
                 files = []
                 for root, dirs, walk_files in os.walk(folder):
-                    files.extend([os.path.join(root, filename) for filename in walk_files])
+                    files.extend([sp(os.path.join(root, filename)) for filename in walk_files])
 
                     # Break if CP wants to shut down
                     if self.shuttingDown():
@@ -134,7 +134,7 @@ class Scanner(Plugin):
                 log.error('Failed getting files from %s: %s', (folder, traceback.format_exc()))
         else:
             check_file_date = False
-            files = [ss(x) for x in files]
+            files = [sp(x) for x in files]
 
 
         for file_path in files:
@@ -454,7 +454,7 @@ class Scanner(Plugin):
                 data['resolution_width'] = meta.get('resolution_width', 720)
                 data['resolution_height'] = meta.get('resolution_height', 480)
                 data['audio_channels'] = meta.get('audio_channels', 2.0)
-                data['aspect'] = meta.get('resolution_width', 720) / meta.get('resolution_height', 480)
+                data['aspect'] = round(float(meta.get('resolution_width', 720)) / meta.get('resolution_height', 480), 2)
             except:
                 log.debug('Error parsing metadata: %s %s', (cur_file, traceback.format_exc()))
                 pass

couchpotato/core/plugins/score/main.py

@@ -1,11 +1,11 @@
-from couchpotato.core.event import addEvent
+from couchpotato.core.event import addEvent, fireEvent
 from couchpotato.core.helpers.encoding import toUnicode
 from couchpotato.core.helpers.variable import getTitle, splitString
 from couchpotato.core.logger import CPLog
 from couchpotato.core.plugins.base import Plugin
 from couchpotato.core.plugins.score.scores import nameScore, nameRatioScore, \
     sizeScore, providerScore, duplicateScore, partialIgnoredScore, namePositionScore, \
-    halfMultipartScore
+    halfMultipartScore, sceneScore
 from couchpotato.environment import Env
 
 log = CPLog(__name__)
@@ -62,4 +62,7 @@ class Score(Plugin):
         if extra_score:
             score += extra_score(nzb)
 
+        # Scene / Nuke scoring
+        score += sceneScore(nzb['name'])
+
         return score

couchpotato/core/plugins/score/scores.py

@@ -1,8 +1,13 @@
 from couchpotato.core.event import fireEvent
 from couchpotato.core.helpers.encoding import simplifyString
 from couchpotato.core.helpers.variable import tryInt
+from couchpotato.core.logger import CPLog
 from couchpotato.environment import Env
 import re
+import traceback
+
+log = CPLog(__name__)
+
 
 name_scores = [
     # Tags
@@ -160,3 +165,38 @@ def halfMultipartScore(nzb_name):
         return -30
 
     return 0
+
+
+def sceneScore(nzb_name):
+
+    check_names = [nzb_name]
+
+    # Match names between "
+    try: check_names.append(re.search(r'([\'"])[^\1]*\1', nzb_name).group(0))
+    except: pass
+
+    # Match longest name between []
+    try: check_names.append(max(re.findall(r'[^[]*\[([^]]*)\]', nzb_name), key = len).strip())
+    except: pass
+
+    for name in check_names:
+
+        # Strip twice, remove possible file extensions
+        name = name.lower().strip(' "\'\.-_\[\]')
+        name = re.sub('\.([a-z0-9]{0,4})$', '', name)
+        name = name.strip(' "\'\.-_\[\]')
+
+        # Make sure year and groupname is in there
+        year = re.findall('(?P<year>19[0-9]{2}|20[0-9]{2})', name)
+        group = re.findall('\-([a-z0-9]+)$', name)
+
+        if len(year) > 0 and len(group) > 0:
+            try:
+                validate = fireEvent('release.validate', name, single = True)
+                if validate and tryInt(validate.get('score')) != 0:
+                    log.debug('Release "%s" scored %s, reason: %s', (nzb_name, validate['score'], validate['reasons']))
+                    return tryInt(validate.get('score'))
+            except:
+                log.error('Failed scoring scene: %s', traceback.format_exc())
+
+    return 0

couchpotato/core/plugins/subtitle/__init__.py

@@ -20,7 +20,7 @@ config = [{
                 },
                 {
                     'name': 'languages',
-                    'description': 'Comma separated, 2 letter country code. Example: en, nl. See the codes at <a href="http://en.wikipedia.org/wiki/List_of_ISO_639-1_codes">on Wikipedia</a>',
+                    'description': ('Comma separated, 2 letter country code.', 'Example: en, nl. See the codes at <a href="http://en.wikipedia.org/wiki/List_of_ISO_639-1_codes">on Wikipedia</a>'),
                 },
 #                {
 #                    'name': 'automatic',

couchpotato/core/plugins/subtitle/main.py

@@ -1,6 +1,6 @@
 from couchpotato import get_session
 from couchpotato.core.event import addEvent, fireEvent
-from couchpotato.core.helpers.encoding import toUnicode
+from couchpotato.core.helpers.encoding import toUnicode, sp
 from couchpotato.core.helpers.variable import splitString
 from couchpotato.core.logger import CPLog
 from couchpotato.core.plugins.base import Plugin
@@ -58,9 +58,9 @@ class Subtitle(Plugin):
 
             for d_sub in downloaded:
                 log.info('Found subtitle (%s): %s', (d_sub.language.alpha2, files))
-                group['files']['subtitle'].append(d_sub.path)
-                group['before_rename'].append(d_sub.path)
-                group['subtitle_language'][d_sub.path] = [d_sub.language.alpha2]
+                group['files']['subtitle'].append(sp(d_sub.path))
+                group['before_rename'].append(sp(d_sub.path))
+                group['subtitle_language'][sp(d_sub.path)] = [d_sub.language.alpha2]
 
             return True
 

couchpotato/core/providers/base.py

@@ -264,14 +264,14 @@ class ResultList(list):
 
     result_ids = None
     provider = None
-    movie = None
+    media = None
     quality = None
 
-    def __init__(self, provider, movie, quality, **kwargs):
+    def __init__(self, provider, media, quality, **kwargs):
 
         self.result_ids = []
         self.provider = provider
-        self.movie = movie
+        self.media = media
         self.quality = quality
         self.kwargs = kwargs
 
@@ -285,13 +285,13 @@ class ResultList(list):
 
         new_result = self.fillResult(result)
 
-        is_correct = fireEvent('searcher.correct_release', new_result, self.movie, self.quality,
+        is_correct = fireEvent('searcher.correct_release', new_result, self.media, self.quality,
                                      imdb_results = self.kwargs.get('imdb_results', False), single = True)
 
         if is_correct and new_result['id'] not in self.result_ids:
             is_correct_weight = float(is_correct)
 
-            new_result['score'] += fireEvent('score.calculate', new_result, self.movie, single = True)
+            new_result['score'] += fireEvent('score.calculate', new_result, self.media, single = True)
 
             old_score = new_result['score']
             new_result['score'] = int(old_score * is_correct_weight)

couchpotato/core/providers/info/_modifier/main.py

@@ -93,11 +93,11 @@ class MovieResultModifier(Plugin):
 
                 for movie in l.movies:
                     if movie.status_id == active_status['id']:
-                        temp['in_wanted'] = fireEvent('movie.get', movie.id, single = True)
+                        temp['in_wanted'] = fireEvent('media.get', movie.id, single = True)
 
                     for release in movie.releases:
                         if release.status_id == done_status['id']:
-                            temp['in_library'] = fireEvent('movie.get', movie.id, single = True)
+                            temp['in_library'] = fireEvent('media.get', movie.id, single = True)
         except:
             log.error('Tried getting more info on searched movies: %s', traceback.format_exc())
 

couchpotato/core/providers/info/couchpotatoapi/main.py

@@ -3,6 +3,7 @@ from couchpotato.core.helpers.encoding import tryUrlencode
 from couchpotato.core.logger import CPLog
 from couchpotato.core.providers.info.base import MovieProvider
 from couchpotato.environment import Env
+import base64
 import time
 
 log = CPLog(__name__)
@@ -11,6 +12,7 @@ log = CPLog(__name__)
 class CouchPotatoApi(MovieProvider):
 
     urls = {
+        'validate': 'https://api.couchpota.to/validate/%s/',
         'search': 'https://api.couchpota.to/search/%s/',
         'info': 'https://api.couchpota.to/info/%s/',
         'is_movie': 'https://api.couchpota.to/ismovie/%s/',
@@ -30,6 +32,8 @@ class CouchPotatoApi(MovieProvider):
         addEvent('movie.suggest', self.getSuggestions)
         addEvent('movie.is_movie', self.isMovie)
 
+        addEvent('release.validate', self.validate)
+
         addEvent('cp.source_url', self.getSourceUrl)
         addEvent('cp.messages', self.getMessages)
 
@@ -51,6 +55,14 @@ class CouchPotatoApi(MovieProvider):
     def search(self, q, limit = 5):
         return self.getJsonData(self.urls['search'] % tryUrlencode(q) + ('?limit=%s' % limit), headers = self.getRequestHeaders())
 
+    def validate(self, name = None):
+
+        if not name:
+            return
+
+        name_enc = base64.b64encode(name)
+        return self.getJsonData(self.urls['validate'] % name_enc, headers = self.getRequestHeaders())
+
     def isMovie(self, identifier = None):
 
         if not identifier:

couchpotato/core/providers/info/omdbapi/main.py

@@ -84,6 +84,10 @@ class OMDBAPI(MovieProvider):
 
             year = tryInt(movie.get('Year', ''))
 
+            actors = {}
+            for actor in splitString(movie.get('Actors', '')):
+                actors[actor] = '' #omdb does not return actor roles
+
             movie_data = {
                 'type': 'movie',
                 'via_imdb': True,
@@ -105,7 +109,7 @@ class OMDBAPI(MovieProvider):
                 'genres': splitString(movie.get('Genre', '')),
                 'directors': splitString(movie.get('Director', '')),
                 'writers': splitString(movie.get('Writer', '')),
-                'actors': splitString(movie.get('Actors', '')),
+                'actor_roles': actors,
             }
             movie_data = dict((k, v) for k, v in movie_data.iteritems() if v)
         except:

couchpotato/core/providers/info/themoviedb/main.py

@@ -92,6 +92,13 @@ class TheMovieDb(MovieProvider):
             poster_original = self.getImage(movie, type = 'poster', size = 'original')
             backdrop_original = self.getImage(movie, type = 'backdrop', size = 'original')
 
+            images = {
+                'poster': [poster] if poster else [],
+                #'backdrop': [backdrop] if backdrop else [],
+                'poster_original': [poster_original] if poster_original else [],
+                'backdrop_original': [backdrop_original] if backdrop_original else [],
+            }
+
             # Genres
             try:
                 genres = [genre.name for genre in movie.genres]
@@ -103,18 +110,22 @@ class TheMovieDb(MovieProvider):
             if not movie.releasedate or year == '1900' or year.lower() == 'none':
                 year = None
 
+            # Gather actors data
+            actors = {}
+            for cast_item in movie.cast:
+                try:
+                    actors[toUnicode(cast_item.name)] = toUnicode(cast_item.character)
+                    images['actor %s' % toUnicode(cast_item.name)] = self.getImage(cast_item, type = 'profile', size = 'original')
+                except:
+                    log.debug('Error getting cast info for %s: %s', (cast_item, traceback.format_exc()))
+
             movie_data = {
                 'type': 'movie',
                 'via_tmdb': True,
                 'tmdb_id': movie.id,
                 'titles': [toUnicode(movie.title)],
                 'original_title': movie.originaltitle,
-                'images': {
-                    'poster': [poster] if poster else [],
-                    #'backdrop': [backdrop] if backdrop else [],
-                    'poster_original': [poster_original] if poster_original else [],
-                    'backdrop_original': [backdrop_original] if backdrop_original else [],
-                },
+                'images': images,
                 'imdb': movie.imdb,
                 'runtime': movie.runtime,
                 'released': str(movie.releasedate),
@@ -122,6 +133,7 @@ class TheMovieDb(MovieProvider):
                 'plot': movie.overview,
                 'genres': genres,
                 'collection': getattr(movie.collection, 'name', None),
+                'actor_roles': actors
             }
 
             movie_data = dict((k, v) for k, v in movie_data.iteritems() if v)
@@ -145,7 +157,7 @@ class TheMovieDb(MovieProvider):
         try:
             image_url = getattr(movie, type).geturl(size = 'original')
         except:
-            log.debug('Failed getting %s.%s for "%s"', (type, size, movie.title))
+            log.debug('Failed getting %s.%s for "%s"', (type, size, movie))
 
         return image_url
 

couchpotato/core/providers/metadata/xbmc/main.py

@@ -65,7 +65,7 @@ class XBMC(MetaDataBase):
                 name = type
 
             try:
-                if data['library'].get(type):
+                if movie_info.get(type):
                     el = SubElement(nfoxml, name)
                     el.text = toUnicode(movie_info.get(type, ''))
             except:
@@ -89,10 +89,18 @@ class XBMC(MetaDataBase):
             genres.text = toUnicode(genre)
 
         # Actors
-        for actor in movie_info.get('actors', []):
-            actors = SubElement(nfoxml, 'actor')
-            name = SubElement(actors, 'name')
-            name.text = toUnicode(actor)
+        for actor_name in movie_info.get('actor_roles', {}):
+            role_name = movie_info['actor_roles'][actor_name]
+
+            actor = SubElement(nfoxml, 'actor')
+            name = SubElement(actor, 'name')
+            name.text = toUnicode(actor_name)
+            if role_name:
+                role = SubElement(actor, 'role')
+                role.text = toUnicode(role_name)
+            if movie_info['images'].get('actor %s' % actor_name, ''):
+                thumb = SubElement(actor, 'thumb')
+                thumb.text = toUnicode(movie_info['images'].get('actor %s' % actor_name))
 
         # Directors
         for director_name in movie_info.get('directors', []):
@@ -112,6 +120,51 @@ class XBMC(MetaDataBase):
             sorttitle = SubElement(nfoxml, 'sorttitle')
             sorttitle.text = '%s %s' % (toUnicode(collection_name), movie_info.get('year'))
 
+        # Images
+        for image_url in movie_info['images']['poster_original']:
+            image = SubElement(nfoxml, 'thumb')
+            image.text = toUnicode(image_url)
+        fanart = SubElement(nfoxml, 'fanart')
+        for image_url in movie_info['images']['backdrop_original']:
+            image = SubElement(fanart, 'thumb')
+            image.text = toUnicode(image_url)
+
+        # Add trailer if found
+        trailer_found = False
+        if data.get('renamed_files'):
+            for  filename in data.get('renamed_files'):
+                if 'trailer' in filename:
+                    trailer = SubElement(nfoxml, 'trailer')
+                    trailer.text = toUnicode(filename)
+                    trailer_found = True
+        if not trailer_found and data['files'].get('trailer'):
+            trailer = SubElement(nfoxml, 'trailer')
+            trailer.text = toUnicode(data['files']['trailer'][0])
+
+        # Add file metadata
+        fileinfo = SubElement(nfoxml, 'fileinfo')
+        streamdetails = SubElement(fileinfo, 'streamdetails')
+
+        # Video data
+        if data['meta_data'].get('video'):
+            video = SubElement(streamdetails, 'video')
+            codec = SubElement(video, 'codec')
+            codec.text = toUnicode(data['meta_data']['video'])
+            aspect = SubElement(video, 'aspect')
+            aspect.text = str(data['meta_data']['aspect'])
+            width = SubElement(video, 'width')
+            width.text = str(data['meta_data']['resolution_width'])
+            height = SubElement(video, 'height')
+            height.text = str(data['meta_data']['resolution_height'])
+
+        # Audio data
+        if data['meta_data'].get('audio'):
+            audio = SubElement(streamdetails, 'audio')
+            codec = SubElement(audio, 'codec')
+            codec.text = toUnicode(data['meta_data'].get('audio'))
+            channels = SubElement(audio, 'channels')
+            channels.text = toUnicode(data['meta_data'].get('audio_channels'))
+
         # Clean up the xml and return it
         nfoxml = xml.dom.minidom.parseString(tostring(nfoxml))
         xml_string = nfoxml.toprettyxml(indent = '  ')

couchpotato/core/providers/nzb/newznab/main.py

@@ -10,6 +10,7 @@ from urllib2 import HTTPError
 from urlparse import urlparse
 import time
 import traceback
+import urllib2
 
 log = CPLog(__name__)
 
@@ -159,7 +160,15 @@ class Newznab(NZBProvider, RSS):
                 return 'try_next'
 
         try:
-            data = self.urlopen(url, show_error = False)
+            # Get final redirected url
+            log.debug('Checking %s for redirects.', url)
+            req = urllib2.Request(url)
+            res = urllib2.urlopen(req)
+            finalurl = res.geturl()
+            if finalurl != url:
+                log.debug('Redirect url used: %s', finalurl)
+
+            data = self.urlopen(finalurl, show_error = False)
             self.limits_reached[host] = False
             return data
         except HTTPError, e:

couchpotato/core/providers/torrent/bithdtv/__init__.py

@@ -4,7 +4,7 @@ def start():
     return BiTHDTV()
 
 config = [{
-    'name': 'BiT-HDTV',
+    'name': 'bithdtv',
     'groups': [
         {
             'tab': 'searcher',

couchpotato/core/providers/torrent/torrentpotato/__init__.py

@@ -0,0 +1,66 @@
+from .main import TorrentPotato
+
+def start():
+    return TorrentPotato()
+
+config = [{
+    'name': 'torrentpotato',
+    'groups': [
+        {
+            'tab': 'searcher',
+            'list': 'torrent_providers',
+            'name': 'TorrentPotato',
+            'order': 10,
+            'description': 'CouchPotato torrent provider. Checkout <a href="https://github.com/RuudBurger/CouchPotatoServer/wiki/CouchPotato-Torrent-Provider">the wiki page about this provider</a> for more info.',
+            'wizard': True,
+            'options': [
+                {
+                    'name': 'enabled',
+                    'type': 'enabler',
+                    'default': False,
+                },
+                {
+                    'name': 'use',
+                    'default': ''
+                },
+                {
+                    'name': 'host',
+                    'default': '',
+                    'description': 'The url path of your TorrentPotato provider.',
+                },
+                {
+                    'name': 'extra_score',
+                    'advanced': True,
+                    'label': 'Extra Score',
+                    'default': '0',
+                    'description': 'Starting score for each release found via this provider.',
+                },
+                {
+                    'name': 'name',
+                    'label': 'Username',
+                    'default': '',
+                },
+                {
+                    'name': 'seed_ratio',
+                    'label': 'Seed ratio',
+                    'default': '1',
+                    'description': 'Will not be (re)moved until this seed ratio is met.',
+                },
+                {
+                    'name': 'seed_time',
+                    'label': 'Seed time',
+                    'default': '40',
+                    'description': 'Will not be (re)moved until this seed time (in hours) is met.',
+                },
+                {
+                    'name': 'pass_key',
+                    'default': ',',
+                    'label': 'Pass Key',
+                    'description': 'Can be found on your profile page',
+                    'type': 'combined',
+                    'combine': ['use', 'host', 'pass_key', 'name', 'seed_ratio', 'seed_time', 'extra_score'],
+                },
+            ],
+        },
+    ],
+}]

couchpotato/core/providers/torrent/torrentpotato/main.py

@@ -0,0 +1,129 @@
+from couchpotato.core.helpers.encoding import tryUrlencode, toUnicode
+from couchpotato.core.helpers.variable import splitString, tryInt, tryFloat
+from couchpotato.core.logger import CPLog
+from couchpotato.core.providers.base import ResultList
+from couchpotato.core.providers.torrent.base import TorrentProvider
+from urlparse import urlparse
+import re
+import traceback
+
+log = CPLog(__name__)
+
+
+class TorrentPotato(TorrentProvider):
+
+    urls = {}
+    limits_reached = {}
+
+    http_time_between_calls = 1 # Seconds
+
+    def search(self, movie, quality):
+        hosts = self.getHosts()
+
+        results = ResultList(self, movie, quality, imdb_results = True)
+
+        for host in hosts:
+            if self.isDisabled(host):
+                continue
+
+            self._searchOnHost(host, movie, quality, results)
+
+        return results
+
+    def _searchOnHost(self, host, movie, quality, results):
+
+        arguments = tryUrlencode({
+            'user': host['name'],
+            'passkey': host['pass_key'],
+            'imdbid': movie['library']['identifier']
+        })
+        url = '%s?%s' % (host['host'], arguments)
+
+        torrents = self.getJsonData(url, cache_timeout = 1800)
+
+        if torrents:
+            try:
+                if torrents.get('error'):
+                    log.error('%s: %s', (torrents.get('error'), host['host']))
+                elif torrents.get('results'):
+                    for torrent in torrents.get('results', []):
+                        results.append({
+                            'id': torrent.get('torrent_id'),
+                            'protocol': 'torrent' if re.match('^(http|https|ftp)://.*$', torrent.get('download_url')) else 'torrent_magnet',
+                            'provider_extra': urlparse(host['host']).hostname or host['host'],
+                            'name': toUnicode(torrent.get('release_name')),
+                            'url': torrent.get('download_url'),
+                            'detail_url': torrent.get('details_url'),
+                            'size': torrent.get('size'),
+                            'score': host['extra_score'],
+                            'seeders': torrent.get('seeders'),
+                            'leechers': torrent.get('leechers'),
+                            'seed_ratio': host['seed_ratio'],
+                            'seed_time': host['seed_time'],
+                        })
+
+            except:
+                log.error('Failed getting results from %s: %s', (host['host'], traceback.format_exc()))
+
+    def getHosts(self):
+
+        uses = splitString(str(self.conf('use')), clean = False)
+        hosts = splitString(self.conf('host'), clean = False)
+        names = splitString(self.conf('name'), clean = False)
+        seed_times = splitString(self.conf('seed_time'), clean = False)
+        seed_ratios = splitString(self.conf('seed_ratio'), clean = False)
+        pass_keys = splitString(self.conf('pass_key'), clean = False)
+        extra_score = splitString(self.conf('extra_score'), clean = False)
+
+        list = []
+        for nr in range(len(hosts)):
+
+            try: key = pass_keys[nr]
+            except: key = ''
+
+            try: host = hosts[nr]
+            except: host = ''
+
+            try: name = names[nr]
+            except: name = ''
+
+            try: ratio = seed_ratios[nr]
+            except: ratio = ''
+
+            try: seed_time = seed_times[nr]
+            except: seed_time = ''
+
+            list.append({
+                'use': uses[nr],
+                'host': host,
+                'name': name,
+                'seed_ratio': tryFloat(ratio),
+                'seed_time': tryInt(seed_time),
+                'pass_key': key,
+                'extra_score': tryInt(extra_score[nr]) if len(extra_score) > nr else 0
+            })
+
+        return list
+
+    def belongsTo(self, url, provider = None, host = None):
+
+        hosts = self.getHosts()
+
+        for host in hosts:
+            result = super(TorrentPotato, self).belongsTo(url, host = host['host'], provider = provider)
+            if result:
+                return result
+
+    def isDisabled(self, host = None):
+        return not self.isEnabled(host)
+
+    def isEnabled(self, host = None):
+
+    # Return true if at least one is enabled and no host is given
+        if host is None:
+            for host in self.getHosts():
+                if self.isEnabled(host):
+                    return True
+            return False
+
+        return TorrentProvider.isEnabled(self) and host['host'] and host['pass_key'] and int(host['use'])

couchpotato/environment.py

@@ -78,6 +78,7 @@ class Env(object):
             return s.get(attr, default = default, section = section, type = type)
 
         # Set setting
+        s.addSection(section)
         s.set(section, attr, value)
         s.save()
 

couchpotato/runner.py

@@ -8,6 +8,7 @@ from couchpotato.core.helpers.variable import getDataDir, tryInt
 from logging import handlers
 from tornado.httpserver import HTTPServer
 from tornado.web import Application, StaticFileHandler, RedirectHandler
+from uuid import uuid4
 import locale
 import logging
 import os.path
@@ -144,7 +145,7 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En
     Env.set('dev', development)
 
     # Disable logging for some modules
-    for logger_name in ['enzyme', 'guessit', 'subliminal', 'apscheduler']:
+    for logger_name in ['enzyme', 'guessit', 'subliminal', 'apscheduler', 'tornado']:
         logging.getLogger(logger_name).setLevel(logging.ERROR)
 
     for logger_name in ['gntp', 'migrate']:
@@ -215,6 +216,10 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En
     Env.set('web_base', web_base)
 
     api_key = Env.setting('api_key')
+    if not api_key:
+        api_key = uuid4().hex
+        Env.setting('api_key', value = api_key)
+
     api_base = r'%sapi/%s/' % (web_base, api_key)
     Env.set('api_base', api_base)
 

couchpotato/static/scripts/couchpotato.js

@@ -11,6 +11,12 @@
 	pages: [],
 	block: [],
 
+	initialize: function(){
+		var self = this;
+
+		self.global_events = {};
+	},
+
 	setup: function(options) {
 		var self = this;
 		self.setOptions(options);
@@ -30,7 +36,7 @@
 		History.addEvent('change', self.openPage.bind(self));
 		self.c.addEvent('click:relay(a[href^=/]:not([target]))', self.pushState.bind(self));
 		self.c.addEvent('click:relay(a[href^=http])', self.openDerefered.bind(self));
-		
+
 		// Check if device is touchenabled
 		self.touch_device = 'ontouchstart' in window || navigator.msMaxTouchPoints;
 		if(self.touch_device)
@@ -55,7 +61,7 @@
 				History.push(url);
 		}
 	},
-	
+
 	isMac: function(){
 		return Browser.Platform.mac
 	},
@@ -111,7 +117,7 @@
 				}
 			})
 		];
-		
+
 		setting_links.each(function(a){
 			self.block.more.addLink(a)
 		});
@@ -336,6 +342,66 @@
 				})
 			)
 		);
+	},
+
+	/*
+	 * Global events
+	 */
+	on: function(name, handle){
+		var self = this;
+
+		if(!self.global_events[name])
+			self.global_events[name] = [];
+
+		self.global_events[name].push(handle);
+
+	},
+
+	trigger: function(name, args, on_complete){
+		var self = this;
+
+		if(!self.global_events[name]){ return; }
+
+		if(!on_complete && typeOf(args) == 'function'){
+			on_complete = args;
+			args = {};
+		}
+
+		// Create parallel callback
+		var callbacks = [];
+		self.global_events[name].each(function(handle, nr){
+
+			callbacks.push(function(callback){
+				var results = handle(args || {});
+				callback(null, results || null);
+			});
+
+		});
+
+		// Fire events
+		async.parallel(callbacks, function(err, results){
+			if(err) p(err);
+
+			if(on_complete)
+				on_complete(results);
+		});
+
+	},
+
+	off: function(name, handle){
+		var self = this;
+
+		if(!self.global_events[name]) return;
+
+		// Remove single
+		if(handle){
+			self.global_events[name] = self.global_events[name].erase(handle);
+		}
+		// Reset full event
+		else {
+			self.global_events[name] = [];
+		}
+
 	}
 
 });
@@ -503,7 +569,7 @@ function randomString(length, extra) {
 					case "string": saveKeyPath(argument.match(/[+-]|[^.]+/g)); break;
 				}
 			});
-			return this.sort(comparer);
+			return this.stableSort(comparer);
 		}
 	});
 

couchpotato/static/scripts/library/Array.stableSort.js

@@ -0,0 +1,56 @@
+/*
+---
+
+script: Array.stableSort.js
+
+description: Add a stable sort algorithm for all browsers
+
+license: MIT-style license.
+
+authors:
+  - Yorick Sijsling
+
+requires:
+  core/1.3: '*'
+
+provides:
+  - [Array.stableSort, Array.mergeSort]
+
+...
+*/
+
+(function() {
+    
+    var defaultSortFunction = function(a, b) {
+        return a > b ? 1 : (a < b ? -1 : 0);
+    }
+    
+    Array.implement({
+        
+        stableSort: function(compare) {
+            // I would love some real feature recognition. Problem is that an unstable algorithm sometimes/often gives the same result as an unstable algorithm.
+            return (Browser.chrome || Browser.firefox2 || Browser.opera9) ? this.mergeSort(compare) : this.sort(compare);
+        },
+        
+        mergeSort: function(compare, token) {
+            compare = compare || defaultSortFunction;
+            if (this.length > 1) {
+                // Split and sort both parts
+                var right = this.splice(Math.floor(this.length / 2)).mergeSort(compare);
+                var left = this.splice(0).mergeSort(compare); // 'this' is now empty.
+                
+                // Merge parts together
+                while (left.length > 0 || right.length > 0) {
+                    this.push(
+                        right.length === 0 ? left.shift()
+                      : left.length === 0 ? right.shift()
+                      : compare(left[0], right[0]) > 0 ? right.shift()
+                      : left.shift());
+                }
+            }
+            return this;
+        }
+    
+    });
+})();
+

couchpotato/static/scripts/library/async.js

@@ -0,0 +1,955 @@
+/*global setImmediate: false, setTimeout: false, console: false */
+(function () {
+
+    var async = {};
+
+    // global on the server, window in the browser
+    var root, previous_async;
+
+    root = this;
+    if (root != null) {
+      previous_async = root.async;
+    }
+
+    async.noConflict = function () {
+        root.async = previous_async;
+        return async;
+    };
+
+    function only_once(fn) {
+        var called = false;
+        return function() {
+            if (called) throw new Error("Callback was already called.");
+            called = true;
+            fn.apply(root, arguments);
+        }
+    }
+
+    //// cross-browser compatiblity functions ////
+
+    var _each = function (arr, iterator) {
+        if (arr.forEach) {
+            return arr.forEach(iterator);
+        }
+        for (var i = 0; i < arr.length; i += 1) {
+            iterator(arr[i], i, arr);
+        }
+    };
+
+    var _map = function (arr, iterator) {
+        if (arr.map) {
+            return arr.map(iterator);
+        }
+        var results = [];
+        _each(arr, function (x, i, a) {
+            results.push(iterator(x, i, a));
+        });
+        return results;
+    };
+
+    var _reduce = function (arr, iterator, memo) {
+        if (arr.reduce) {
+            return arr.reduce(iterator, memo);
+        }
+        _each(arr, function (x, i, a) {
+            memo = iterator(memo, x, i, a);
+        });
+        return memo;
+    };
+
+    var _keys = function (obj) {
+        if (Object.keys) {
+            return Object.keys(obj);
+        }
+        var keys = [];
+        for (var k in obj) {
+            if (obj.hasOwnProperty(k)) {
+                keys.push(k);
+            }
+        }
+        return keys;
+    };
+
+    //// exported async module functions ////
+
+    //// nextTick implementation with browser-compatible fallback ////
+    if (typeof process === 'undefined' || !(process.nextTick)) {
+        if (typeof setImmediate === 'function') {
+            async.nextTick = function (fn) {
+                // not a direct alias for IE10 compatibility
+                setImmediate(fn);
+            };
+            async.setImmediate = async.nextTick;
+        }
+        else {
+            async.nextTick = function (fn) {
+                setTimeout(fn, 0);
+            };
+            async.setImmediate = async.nextTick;
+        }
+    }
+    else {
+        async.nextTick = process.nextTick;
+        if (typeof setImmediate !== 'undefined') {
+            async.setImmediate = setImmediate;
+        }
+        else {
+            async.setImmediate = async.nextTick;
+        }
+    }
+
+    async.each = function (arr, iterator, callback) {
+        callback = callback || function () {};
+        if (!arr.length) {
+            return callback();
+        }
+        var completed = 0;
+        _each(arr, function (x) {
+            iterator(x, only_once(function (err) {
+                if (err) {
+                    callback(err);
+                    callback = function () {};
+                }
+                else {
+                    completed += 1;
+                    if (completed >= arr.length) {
+                        callback(null);
+                    }
+                }
+            }));
+        });
+    };
+    async.forEach = async.each;
+
+    async.eachSeries = function (arr, iterator, callback) {
+        callback = callback || function () {};
+        if (!arr.length) {
+            return callback();
+        }
+        var completed = 0;
+        var iterate = function () {
+            iterator(arr[completed], function (err) {
+                if (err) {
+                    callback(err);
+                    callback = function () {};
+                }
+                else {
+                    completed += 1;
+                    if (completed >= arr.length) {
+                        callback(null);
+                    }
+                    else {
+                        iterate();
+                    }
+                }
+            });
+        };
+        iterate();
+    };
+    async.forEachSeries = async.eachSeries;
+
+    async.eachLimit = function (arr, limit, iterator, callback) {
+        var fn = _eachLimit(limit);
+        fn.apply(null, [arr, iterator, callback]);
+    };
+    async.forEachLimit = async.eachLimit;
+
+    var _eachLimit = function (limit) {
+
+        return function (arr, iterator, callback) {
+            callback = callback || function () {};
+            if (!arr.length || limit <= 0) {
+                return callback();
+            }
+            var completed = 0;
+            var started = 0;
+            var running = 0;
+
+            (function replenish () {
+                if (completed >= arr.length) {
+                    return callback();
+                }
+
+                while (running < limit && started < arr.length) {
+                    started += 1;
+                    running += 1;
+                    iterator(arr[started - 1], function (err) {
+                        if (err) {
+                            callback(err);
+                            callback = function () {};
+                        }
+                        else {
+                            completed += 1;
+                            running -= 1;
+                            if (completed >= arr.length) {
+                                callback();
+                            }
+                            else {
+                                replenish();
+                            }
+                        }
+                    });
+                }
+            })();
+        };
+    };
+
+
+    var doParallel = function (fn) {
+        return function () {
+            var args = Array.prototype.slice.call(arguments);
+            return fn.apply(null, [async.each].concat(args));
+        };
+    };
+    var doParallelLimit = function(limit, fn) {
+        return function () {
+            var args = Array.prototype.slice.call(arguments);
+            return fn.apply(null, [_eachLimit(limit)].concat(args));
+        };
+    };
+    var doSeries = function (fn) {
+        return function () {
+            var args = Array.prototype.slice.call(arguments);
+            return fn.apply(null, [async.eachSeries].concat(args));
+        };
+    };
+
+
+    var _asyncMap = function (eachfn, arr, iterator, callback) {
+        var results = [];
+        arr = _map(arr, function (x, i) {
+            return {index: i, value: x};
+        });
+        eachfn(arr, function (x, callback) {
+            iterator(x.value, function (err, v) {
+                results[x.index] = v;
+                callback(err);
+            });
+        }, function (err) {
+            callback(err, results);
+        });
+    };
+    async.map = doParallel(_asyncMap);
+    async.mapSeries = doSeries(_asyncMap);
+    async.mapLimit = function (arr, limit, iterator, callback) {
+        return _mapLimit(limit)(arr, iterator, callback);
+    };
+
+    var _mapLimit = function(limit) {
+        return doParallelLimit(limit, _asyncMap);
+    };
+
+    // reduce only has a series version, as doing reduce in parallel won't
+    // work in many situations.
+    async.reduce = function (arr, memo, iterator, callback) {
+        async.eachSeries(arr, function (x, callback) {
+            iterator(memo, x, function (err, v) {
+                memo = v;
+                callback(err);
+            });
+        }, function (err) {
+            callback(err, memo);
+        });
+    };
+    // inject alias
+    async.inject = async.reduce;
+    // foldl alias
+    async.foldl = async.reduce;
+
+    async.reduceRight = function (arr, memo, iterator, callback) {
+        var reversed = _map(arr, function (x) {
+            return x;
+        }).reverse();
+        async.reduce(reversed, memo, iterator, callback);
+    };
+    // foldr alias
+    async.foldr = async.reduceRight;
+
+    var _filter = function (eachfn, arr, iterator, callback) {
+        var results = [];
+        arr = _map(arr, function (x, i) {
+            return {index: i, value: x};
+        });
+        eachfn(arr, function (x, callback) {
+            iterator(x.value, function (v) {
+                if (v) {
+                    results.push(x);
+                }
+                callback();
+            });
+        }, function (err) {
+            callback(_map(results.sort(function (a, b) {
+                return a.index - b.index;
+            }), function (x) {
+                return x.value;
+            }));
+        });
+    };
+    async.filter = doParallel(_filter);
+    async.filterSeries = doSeries(_filter);
+    // select alias
+    async.select = async.filter;
+    async.selectSeries = async.filterSeries;
+
+    var _reject = function (eachfn, arr, iterator, callback) {
+        var results = [];
+        arr = _map(arr, function (x, i) {
+            return {index: i, value: x};
+        });
+        eachfn(arr, function (x, callback) {
+            iterator(x.value, function (v) {
+                if (!v) {
+                    results.push(x);
+                }
+                callback();
+            });
+        }, function (err) {
+            callback(_map(results.sort(function (a, b) {
+                return a.index - b.index;
+            }), function (x) {
+                return x.value;
+            }));
+        });
+    };
+    async.reject = doParallel(_reject);
+    async.rejectSeries = doSeries(_reject);
+
+    var _detect = function (eachfn, arr, iterator, main_callback) {
+        eachfn(arr, function (x, callback) {
+            iterator(x, function (result) {
+                if (result) {
+                    main_callback(x);
+                    main_callback = function () {};
+                }
+                else {
+                    callback();
+                }
+            });
+        }, function (err) {
+            main_callback();
+        });
+    };
+    async.detect = doParallel(_detect);
+    async.detectSeries = doSeries(_detect);
+
+    async.some = function (arr, iterator, main_callback) {
+        async.each(arr, function (x, callback) {
+            iterator(x, function (v) {
+                if (v) {
+                    main_callback(true);
+                    main_callback = function () {};
+                }
+                callback();
+            });
+        }, function (err) {
+            main_callback(false);
+        });
+    };
+    // any alias
+    async.any = async.some;
+
+    async.every = function (arr, iterator, main_callback) {
+        async.each(arr, function (x, callback) {
+            iterator(x, function (v) {
+                if (!v) {
+                    main_callback(false);
+                    main_callback = function () {};
+                }
+                callback();
+            });
+        }, function (err) {
+            main_callback(true);
+        });
+    };
+    // all alias
+    async.all = async.every;
+
+    async.sortBy = function (arr, iterator, callback) {
+        async.map(arr, function (x, callback) {
+            iterator(x, function (err, criteria) {
+                if (err) {
+                    callback(err);
+                }
+                else {
+                    callback(null, {value: x, criteria: criteria});
+                }
+            });
+        }, function (err, results) {
+            if (err) {
+                return callback(err);
+            }
+            else {
+                var fn = function (left, right) {
+                    var a = left.criteria, b = right.criteria;
+                    return a < b ? -1 : a > b ? 1 : 0;
+                };
+                callback(null, _map(results.sort(fn), function (x) {
+                    return x.value;
+                }));
+            }
+        });
+    };
+
+    async.auto = function (tasks, callback) {
+        callback = callback || function () {};
+        var keys = _keys(tasks);
+        if (!keys.length) {
+            return callback(null);
+        }
+
+        var results = {};
+
+        var listeners = [];
+        var addListener = function (fn) {
+            listeners.unshift(fn);
+        };
+        var removeListener = function (fn) {
+            for (var i = 0; i < listeners.length; i += 1) {
+                if (listeners[i] === fn) {
+                    listeners.splice(i, 1);
+                    return;
+                }
+            }
+        };
+        var taskComplete = function () {
+            _each(listeners.slice(0), function (fn) {
+                fn();
+            });
+        };
+
+        addListener(function () {
+            if (_keys(results).length === keys.length) {
+                callback(null, results);
+                callback = function () {};
+            }
+        });
+
+        _each(keys, function (k) {
+            var task = (tasks[k] instanceof Function) ? [tasks[k]]: tasks[k];
+            var taskCallback = function (err) {
+                var args = Array.prototype.slice.call(arguments, 1);
+                if (args.length <= 1) {
+                    args = args[0];
+                }
+                if (err) {
+                    var safeResults = {};
+                    _each(_keys(results), function(rkey) {
+                        safeResults[rkey] = results[rkey];
+                    });
+                    safeResults[k] = args;
+                    callback(err, safeResults);
+                    // stop subsequent errors hitting callback multiple times
+                    callback = function () {};
+                }
+                else {
+                    results[k] = args;
+                    async.setImmediate(taskComplete);
+                }
+            };
+            var requires = task.slice(0, Math.abs(task.length - 1)) || [];
+            var ready = function () {
+                return _reduce(requires, function (a, x) {
+                    return (a && results.hasOwnProperty(x));
+                }, true) && !results.hasOwnProperty(k);
+            };
+            if (ready()) {
+                task[task.length - 1](taskCallback, results);
+            }
+            else {
+                var listener = function () {
+                    if (ready()) {
+                        removeListener(listener);
+                        task[task.length - 1](taskCallback, results);
+                    }
+                };
+                addListener(listener);
+            }
+        });
+    };
+
+    async.waterfall = function (tasks, callback) {
+        callback = callback || function () {};
+        if (tasks.constructor !== Array) {
+          var err = new Error('First argument to waterfall must be an array of functions');
+          return callback(err);
+        }
+        if (!tasks.length) {
+            return callback();
+        }
+        var wrapIterator = function (iterator) {
+            return function (err) {
+                if (err) {
+                    callback.apply(null, arguments);
+                    callback = function () {};
+                }
+                else {
+                    var args = Array.prototype.slice.call(arguments, 1);
+                    var next = iterator.next();
+                    if (next) {
+                        args.push(wrapIterator(next));
+                    }
+                    else {
+                        args.push(callback);
+                    }
+                    async.setImmediate(function () {
+                        iterator.apply(null, args);
+                    });
+                }
+            };
+        };
+        wrapIterator(async.iterator(tasks))();
+    };
+
+    var _parallel = function(eachfn, tasks, callback) {
+        callback = callback || function () {};
+        if (tasks.constructor === Array) {
+            eachfn.map(tasks, function (fn, callback) {
+                if (fn) {
+                    fn(function (err) {
+                        var args = Array.prototype.slice.call(arguments, 1);
+                        if (args.length <= 1) {
+                            args = args[0];
+                        }
+                        callback.call(null, err, args);
+                    });
+                }
+            }, callback);
+        }
+        else {
+            var results = {};
+            eachfn.each(_keys(tasks), function (k, callback) {
+                tasks[k](function (err) {
+                    var args = Array.prototype.slice.call(arguments, 1);
+                    if (args.length <= 1) {
+                        args = args[0];
+                    }
+                    results[k] = args;
+                    callback(err);
+                });
+            }, function (err) {
+                callback(err, results);
+            });
+        }
+    };
+
+    async.parallel = function (tasks, callback) {
+        _parallel({ map: async.map, each: async.each }, tasks, callback);
+    };
+
+    async.parallelLimit = function(tasks, limit, callback) {
+        _parallel({ map: _mapLimit(limit), each: _eachLimit(limit) }, tasks, callback);
+    };
+
+    async.series = function (tasks, callback) {
+        callback = callback || function () {};
+        if (tasks.constructor === Array) {
+            async.mapSeries(tasks, function (fn, callback) {
+                if (fn) {
+                    fn(function (err) {
+                        var args = Array.prototype.slice.call(arguments, 1);
+                        if (args.length <= 1) {
+                            args = args[0];
+                        }
+                        callback.call(null, err, args);
+                    });
+                }
+            }, callback);
+        }
+        else {
+            var results = {};
+            async.eachSeries(_keys(tasks), function (k, callback) {
+                tasks[k](function (err) {
+                    var args = Array.prototype.slice.call(arguments, 1);
+                    if (args.length <= 1) {
+                        args = args[0];
+                    }
+                    results[k] = args;
+                    callback(err);
+                });
+            }, function (err) {
+                callback(err, results);
+            });
+        }
+    };
+
+    async.iterator = function (tasks) {
+        var makeCallback = function (index) {
+            var fn = function () {
+                if (tasks.length) {
+                    tasks[index].apply(null, arguments);
+                }
+                return fn.next();
+            };
+            fn.next = function () {
+                return (index < tasks.length - 1) ? makeCallback(index + 1): null;
+            };
+            return fn;
+        };
+        return makeCallback(0);
+    };
+
+    async.apply = function (fn) {
+        var args = Array.prototype.slice.call(arguments, 1);
+        return function () {
+            return fn.apply(
+                null, args.concat(Array.prototype.slice.call(arguments))
+            );
+        };
+    };
+
+    var _concat = function (eachfn, arr, fn, callback) {
+        var r = [];
+        eachfn(arr, function (x, cb) {
+            fn(x, function (err, y) {
+                r = r.concat(y || []);
+                cb(err);
+            });
+        }, function (err) {
+            callback(err, r);
+        });
+    };
+    async.concat = doParallel(_concat);
+    async.concatSeries = doSeries(_concat);
+
+    async.whilst = function (test, iterator, callback) {
+        if (test()) {
+            iterator(function (err) {
+                if (err) {
+                    return callback(err);
+                }
+                async.whilst(test, iterator, callback);
+            });
+        }
+        else {
+            callback();
+        }
+    };
+
+    async.doWhilst = function (iterator, test, callback) {
+        iterator(function (err) {
+            if (err) {
+                return callback(err);
+            }
+            if (test()) {
+                async.doWhilst(iterator, test, callback);
+            }
+            else {
+                callback();
+            }
+        });
+    };
+
+    async.until = function (test, iterator, callback) {
+        if (!test()) {
+            iterator(function (err) {
+                if (err) {
+                    return callback(err);
+                }
+                async.until(test, iterator, callback);
+            });
+        }
+        else {
+            callback();
+        }
+    };
+
+    async.doUntil = function (iterator, test, callback) {
+        iterator(function (err) {
+            if (err) {
+                return callback(err);
+            }
+            if (!test()) {
+                async.doUntil(iterator, test, callback);
+            }
+            else {
+                callback();
+            }
+        });
+    };
+
+    async.queue = function (worker, concurrency) {
+        if (concurrency === undefined) {
+            concurrency = 1;
+        }
+        function _insert(q, data, pos, callback) {
+          if(data.constructor !== Array) {
+              data = [data];
+          }
+          _each(data, function(task) {
+              var item = {
+                  data: task,
+                  callback: typeof callback === 'function' ? callback : null
+              };
+
+              if (pos) {
+                q.tasks.unshift(item);
+              } else {
+                q.tasks.push(item);
+              }
+
+              if (q.saturated && q.tasks.length === concurrency) {
+                  q.saturated();
+              }
+              async.setImmediate(q.process);
+          });
+        }
+
+        var workers = 0;
+        var q = {
+            tasks: [],
+            concurrency: concurrency,
+            saturated: null,
+            empty: null,
+            drain: null,
+            push: function (data, callback) {
+              _insert(q, data, false, callback);
+            },
+            unshift: function (data, callback) {
+              _insert(q, data, true, callback);
+            },
+            process: function () {
+                if (workers < q.concurrency && q.tasks.length) {
+                    var task = q.tasks.shift();
+                    if (q.empty && q.tasks.length === 0) {
+                        q.empty();
+                    }
+                    workers += 1;
+                    var next = function () {
+                        workers -= 1;
+                        if (task.callback) {
+                            task.callback.apply(task, arguments);
+                        }
+                        if (q.drain && q.tasks.length + workers === 0) {
+                            q.drain();
+                        }
+                        q.process();
+                    };
+                    var cb = only_once(next);
+                    worker(task.data, cb);
+                }
+            },
+            length: function () {
+                return q.tasks.length;
+            },
+            running: function () {
+                return workers;
+            }
+        };
+        return q;
+    };
+
+    async.cargo = function (worker, payload) {
+        var working     = false,
+            tasks       = [];
+
+        var cargo = {
+            tasks: tasks,
+            payload: payload,
+            saturated: null,
+            empty: null,
+            drain: null,
+            push: function (data, callback) {
+                if(data.constructor !== Array) {
+                    data = [data];
+                }
+                _each(data, function(task) {
+                    tasks.push({
+                        data: task,
+                        callback: typeof callback === 'function' ? callback : null
+                    });
+                    if (cargo.saturated && tasks.length === payload) {
+                        cargo.saturated();
+                    }
+                });
+                async.setImmediate(cargo.process);
+            },
+            process: function process() {
+                if (working) return;
+                if (tasks.length === 0) {
+                    if(cargo.drain) cargo.drain();
+                    return;
+                }
+
+                var ts = typeof payload === 'number'
+                            ? tasks.splice(0, payload)
+                            : tasks.splice(0);
+
+                var ds = _map(ts, function (task) {
+                    return task.data;
+                });
+
+                if(cargo.empty) cargo.empty();
+                working = true;
+                worker(ds, function () {
+                    working = false;
+
+                    var args = arguments;
+                    _each(ts, function (data) {
+                        if (data.callback) {
+                            data.callback.apply(null, args);
+                        }
+                    });
+
+                    process();
+                });
+            },
+            length: function () {
+                return tasks.length;
+            },
+            running: function () {
+                return working;
+            }
+        };
+        return cargo;
+    };
+
+    var _console_fn = function (name) {
+        return function (fn) {
+            var args = Array.prototype.slice.call(arguments, 1);
+            fn.apply(null, args.concat([function (err) {
+                var args = Array.prototype.slice.call(arguments, 1);
+                if (typeof console !== 'undefined') {
+                    if (err) {
+                        if (console.error) {
+                            console.error(err);
+                        }
+                    }
+                    else if (console[name]) {
+                        _each(args, function (x) {
+                            console[name](x);
+                        });
+                    }
+                }
+            }]));
+        };
+    };
+    async.log = _console_fn('log');
+    async.dir = _console_fn('dir');
+    /*async.info = _console_fn('info');
+    async.warn = _console_fn('warn');
+    async.error = _console_fn('error');*/
+
+    async.memoize = function (fn, hasher) {
+        var memo = {};
+        var queues = {};
+        hasher = hasher || function (x) {
+            return x;
+        };
+        var memoized = function () {
+            var args = Array.prototype.slice.call(arguments);
+            var callback = args.pop();
+            var key = hasher.apply(null, args);
+            if (key in memo) {
+                callback.apply(null, memo[key]);
+            }
+            else if (key in queues) {
+                queues[key].push(callback);
+            }
+            else {
+                queues[key] = [callback];
+                fn.apply(null, args.concat([function () {
+                    memo[key] = arguments;
+                    var q = queues[key];
+                    delete queues[key];
+                    for (var i = 0, l = q.length; i < l; i++) {
+                      q[i].apply(null, arguments);
+                    }
+                }]));
+            }
+        };
+        memoized.memo = memo;
+        memoized.unmemoized = fn;
+        return memoized;
+    };
+
+    async.unmemoize = function (fn) {
+      return function () {
+        return (fn.unmemoized || fn).apply(null, arguments);
+      };
+    };
+
+    async.times = function (count, iterator, callback) {
+        var counter = [];
+        for (var i = 0; i < count; i++) {
+            counter.push(i);
+        }
+        return async.map(counter, iterator, callback);
+    };
+
+    async.timesSeries = function (count, iterator, callback) {
+        var counter = [];
+        for (var i = 0; i < count; i++) {
+            counter.push(i);
+        }
+        return async.mapSeries(counter, iterator, callback);
+    };
+
+    async.compose = function (/* functions... */) {
+        var fns = Array.prototype.reverse.call(arguments);
+        return function () {
+            var that = this;
+            var args = Array.prototype.slice.call(arguments);
+            var callback = args.pop();
+            async.reduce(fns, args, function (newargs, fn, cb) {
+                fn.apply(that, newargs.concat([function () {
+                    var err = arguments[0];
+                    var nextargs = Array.prototype.slice.call(arguments, 1);
+                    cb(err, nextargs);
+                }]))
+            },
+            function (err, results) {
+                callback.apply(that, [err].concat(results));
+            });
+        };
+    };
+
+    var _applyEach = function (eachfn, fns /*args...*/) {
+        var go = function () {
+            var that = this;
+            var args = Array.prototype.slice.call(arguments);
+            var callback = args.pop();
+            return eachfn(fns, function (fn, cb) {
+                fn.apply(that, args.concat([cb]));
+            },
+            callback);
+        };
+        if (arguments.length > 2) {
+            var args = Array.prototype.slice.call(arguments, 2);
+            return go.apply(this, args);
+        }
+        else {
+            return go;
+        }
+    };
+    async.applyEach = doParallel(_applyEach);
+    async.applyEachSeries = doSeries(_applyEach);
+
+    async.forever = function (fn, callback) {
+        function next(err) {
+            if (err) {
+                if (callback) {
+                    return callback(err);
+                }
+                throw err;
+            }
+            fn(next);
+        }
+        next();
+    };
+
+    // AMD / RequireJS
+    if (typeof define !== 'undefined' && define.amd) {
+        define([], function () {
+            return async;
+        });
+    }
+    // Node.js
+    else if (typeof module !== 'undefined' && module.exports) {
+        module.exports = async;
+    }
+    // included directly via <script> tag
+    else {
+        root.async = async;
+    }
+
+}());

couchpotato/static/scripts/library/mootools_more.js

@@ -1,5 +1,5 @@
 // MooTools: the javascript framework.
-// Load this file's selection again by visiting: http://mootools.net/more/43db227db7a621ebb062ee621432ae3d 
+// Load this file's selection again by visiting: http://mootools.net/more/0f75cfbac1aabbedaba7630beef8d10c 
 // Or build this file again with packager using: packager build More/Events.Pseudos More/Date More/Date.Extras More/Element.Forms More/Element.Position More/Element.Shortcuts More/Fx.Scroll More/Fx.Slide More/Sortables More/Request.JSONP More/Request.Periodical
 /*
 ---

couchpotato/static/scripts/page/home.js

@@ -57,6 +57,19 @@ Page.Home = new Class({
 			'limit': null,
 			'onLoaded': function(){
 				self.chain.callChain();
+			},
+			'onMovieAdded': function(notification){
+
+				// Track movie added
+				var after_search = function(data){
+					if(notification.data.id != data.data.id) return;
+
+					// Force update after search
+					self.available_list.update();
+					App.off('movie.searcher.ended', after_search);
+				}
+				App.on('movie.searcher.ended', after_search);
+
 			}
 		});
 

couchpotato/static/scripts/page/manage.js

@@ -102,6 +102,8 @@ Page.Manage = new Class({
 						}
 					}
 					else {
+						// Capture progress so we can use it in our *each* closure
+						var progress = json.progress
 
 						// Don't add loader when page is loading still
 						if(!self.list.navigation)
@@ -112,10 +114,13 @@ Page.Manage = new Class({
 
 						self.progress_container.empty();
 
-						Object.each(json.progress, function(progress, folder){
+						var sorted_table = self.parseProgress(json.progress)
+
+						sorted_table.each(function(folder){
+							var folder_progress = progress[folder]
 							new Element('div').adopt(
 								new Element('span.folder', {'text': folder}),
-								new Element('span.percentage', {'text': progress.total ? (((progress.total-progress.to_go)/progress.total)*100).round() + '%' : '0%'})
+								new Element('span.percentage', {'text': folder_progress.total ? (((folder_progress.total-folder_progress.to_go)/folder_progress.total)*100).round() + '%' : '0%'})
 							).inject(self.progress_container)
 						});
 
@@ -124,7 +129,17 @@ Page.Manage = new Class({
 			})
 
 		}, 1000);
+	},
 
+	parseProgress: function (progress_object) {
+		var folder, temp_array = [];
+
+		for (folder in progress_object) {
+			if (progress_object.hasOwnProperty(folder)) {
+				temp_array.push(folder)
+			}
+		}
+		return temp_array.stableSort()
 	}
 
 });

couchpotato/static/scripts/page/settings.js

@@ -111,6 +111,10 @@ Page.Settings = new Class({
 		Cookie.write('advanced_toggle_checked', +self.advanced_toggle.checked, {'duration': 365});
 	},
 
+    sortByOrder: function(a, b){
+			return (a.order || 100) - (b.order || 100)
+	},
+
 	create: function(json){
 		var self = this;
 
@@ -141,13 +145,11 @@ Page.Settings = new Class({
 			options.include(section);
 		});
 
-		options.sort(function(a, b){
-			return (a.order || 100) - (b.order || 100)
-		}).each(function(section){
+		options.stableSort(self.sortByOrder).each(function(section){
 			var section_name = section.section_name;
 
 			// Add groups to content
-			section.groups.sortBy('order').each(function(group){
+			section.groups.stableSort(self.sortByOrder).each(function(group){
 				if(group.hidden) return;
 
 				if(self.wizard_only && !group.wizard)
@@ -184,9 +186,7 @@ Page.Settings = new Class({
 				}
 
 				// Add options to group
-				group.options.sort(function(a, b){
-					return (a.order || 100) - (b.order || 100)
-				}).each(function(option){
+				group.options.stableSort(self.sortByOrder).each(function(option){
 					if(option.hidden) return;
 					var class_name = (option.type || 'string').capitalize();
 					var input = new Option[class_name](section_name, option.name, self.getValue(section_name, option.name), option);
@@ -265,16 +265,27 @@ Page.Settings = new Class({
 	},
 
 	createGroup: function(group){
+
+		if((typeOf(group.description) == 'array')){
+			var hint = new Element('span.hint.more_hint', {
+				'html': group.description[0]
+			});
+
+			createTooltip(group.description[1]).inject(hint, 'top');
+		}
+		else {
+			var hint = new Element('span.hint', {
+				'html': group.description || ''
+			})
+		}
+
+
 		return new Element('fieldset', {
 			'class': (group.advanced ? 'inlineLabels advanced' : 'inlineLabels') + ' group_' + (group.name || '') + ' subtab_' + (group.subtab || '')
-		}).adopt(
+		}).grab(
 				new Element('h2', {
 					'text': group.label || (group.name).capitalize()
-				}).adopt(
-						new Element('span.hint', {
-							'html': group.description || ''
-						})
-					)
+				}).grab(hint)
 			);
 	},
 
@@ -343,10 +354,22 @@ var OptionBase = new Class({
 
 	createHint: function(){
 		var self = this;
-		if(self.options.description)
-			new Element('p.formHint', {
-				'html': self.options.description
-			}).inject(self.el);
+		if(self.options.description){
+
+
+			if((typeOf(self.options.description) == 'array')){
+				var hint = new Element('p.formHint.more_hint', {
+					'html': self.options.description[0]
+				}).inject(self.el);
+
+				createTooltip(self.options.description[1]).inject(hint, 'top');
+			}
+			else {
+				var hint = new Element('p.formHint', {
+					'html': self.options.description || ''
+				}).inject(self.el)
+			}
+		}
 	},
 
 	afterInject: function(){
@@ -1412,4 +1435,25 @@ Option.Combined = new Class({
 		self.saveCombined();
 	}
 
-});
+});
+
+var createTooltip = function(description){
+
+	var tip = new Element('div.tooltip', {
+			'events': {
+				'mouseenter': function(){
+					tip.addClass('shown')
+				},
+				'mouseleave': function(){
+					tip.removeClass('shown')
+				}
+			}
+		}).adopt(
+			new Element('a.icon2.info'),
+			new Element('div.tip', {
+				'html': description
+			})
+		);
+
+	return tip;
+}

couchpotato/static/style/settings.css

@@ -130,7 +130,7 @@
 		.page .ctrlHolder.focused:first-child, .page .ctrlHolder:first-child{ background-color: transparent; }
 
 		.page .ctrlHolder .formHint {
-			width: 47%;
+			width: 46%;
 			margin: -18px 0;
             color: #fff !important;
 			display: inline-block;
@@ -547,12 +547,31 @@
 		.page .combined_table .head abbr:first-child {
 			display: none;
 		}
-		.page .combined_table .head abbr.host {
-			margin-right: 190px;
+		.page .combined_table .head abbr.host { margin-right: 120px; }
+			.page .combined_table input.host { width: 140px; }
+		.page .section_newznab .combined_table .head abbr.host { margin-right: 200px; }
+			.page .section_newznab .combined_table input.host { width: 220px; }
+			
+		.page .combined_table .head abbr.name { margin-right: 57px; }
+			.page .combined_table input.name { width: 120px; }
+		.page .combined_table .head abbr.api_key { margin-right: 75px; }
+		
+		.page .combined_table .head abbr.pass_key { margin-right: 71px; }
+			.page .combined_table input.pass_key { width: 113px; }
+			
+		.page .section_newznab .combined_table .head abbr.api_key { margin-right: 185px; }
+			.page .section_newznab .combined_table input.api_key { width: 223px; }
+			
+		.page .combined_table .seed_ratio,
+		.page .combined_table .seed_time {
+			width: 70px;
+			text-align: center;
+			margin-left: 10px;
 		}
-		.page .combined_table .head abbr.api_key {
-			margin-right: 171px;
+		.page .combined_table .seed_time {
+			margin-right: 10px;
 		}
+		
 		.page .combined_table .head .extra_score,
 		.page .combined_table .extra_score {
 			width: 70px;
@@ -701,4 +720,46 @@
 .active .group_imdb_automation:not(.disabled) {
 	background: url('../images/imdb_watchlist.png') no-repeat right 50px;
 	min-height: 210px;
-}
+}
+
+
+.tooltip {
+	position: absolute;
+	right: 0px;
+	width: 30px;
+	height: 30px;
+}
+
+	.tooltip > a {
+		opacity: .3;
+		font-size: 11px;
+		cursor: pointer;
+	}
+	
+		.tooltip:hover > a {
+			opacity: 1;
+		}
+
+	.tooltip div {
+		background: #FFF;
+		color: #000;
+		padding: 10px;
+		width: 380px;
+		z-index: 200;
+		position: absolute;
+		transition: all .4s cubic-bezier(0.9,0,0.1,1);
+		margin-top: 40px;
+		right: 0;
+		opacity: 0;
+		visibility: hidden;
+	}
+	
+		.tooltip.shown div {
+			margin-top: 10px;
+			opacity: 1;
+			visibility: visible;
+		}
+		
+			.tooltip div a {
+				color: #5b9bd1;
+			}

couchpotato/templates/index.html

@@ -4,6 +4,8 @@
 	<head>
 	    <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no"/>
 		<meta name="apple-mobile-web-app-capable" content="yes">
+		<meta name="mobile-web-app-capable" content="yes">
+
 
 		{% for url in fireEvent('clientscript.get_styles', as_html = True, location = 'front', single = True) %}
 		<link rel="stylesheet" href="{{ Env.get('web_base') }}{{ url }}" type="text/css">{% end %}

init/fedora

@@ -1,3 +1,5 @@
+#!/bin/sh
+#
 ### BEGIN INIT INFO
 # Provides:          CouchPotato application instance
 # Required-Start:    $all

libs/apscheduler/__init__.py

@@ -1,3 +1,3 @@
-version_info = (2, 0, 2)
+version_info = (2, 1, 1)
 version = '.'.join(str(n) for n in version_info[:3])
-release = version + ''.join(str(n) for n in version_info[3:])
+release = '.'.join(str(n) for n in version_info)

libs/apscheduler/job.py

@@ -16,22 +16,25 @@ class MaxInstancesReachedError(Exception):
 class Job(object):
     """
     Encapsulates the actual Job along with its metadata. Job instances
-    are created by the scheduler when adding jobs, and it should not be
-    directly instantiated.
+    are created by the scheduler when adding jobs, and should not be
+    directly instantiated. These options can be set when adding jobs
+    to the scheduler (see :ref:`job_options`).
 
-    :param trigger: trigger that determines the execution times
-    :param func: callable to call when the trigger is triggered
-    :param args: list of positional arguments to call func with
-    :param kwargs: dict of keyword arguments to call func with
-    :param name: name of the job (optional)
-    :param misfire_grace_time: seconds after the designated run time that
+    :var trigger: trigger that determines the execution times
+    :var func: callable to call when the trigger is triggered
+    :var args: list of positional arguments to call func with
+    :var kwargs: dict of keyword arguments to call func with
+    :var name: name of the job
+    :var misfire_grace_time: seconds after the designated run time that
         the job is still allowed to be run
-    :param coalesce: run once instead of many times if the scheduler determines
+    :var coalesce: run once instead of many times if the scheduler determines
         that the job should be run more than once in succession
-    :param max_runs: maximum number of times this job is allowed to be
+    :var max_runs: maximum number of times this job is allowed to be
         triggered
-    :param max_instances: maximum number of concurrently running
+    :var max_instances: maximum number of concurrently running
         instances allowed for this job
+    :var runs: number of times this job has been triggered
+    :var instances: number of concurrently running instances of this job
     """
     id = None
     next_run_time = None
@@ -130,5 +133,5 @@ class Job(object):
         return '<Job (name=%s, trigger=%s)>' % (self.name, repr(self.trigger))
 
     def __str__(self):
-        return '%s (trigger: %s, next run at: %s)' % (self.name,
-            str(self.trigger), str(self.next_run_time))
+        return '%s (trigger: %s, next run at: %s)' % (
+            self.name, str(self.trigger), str(self.next_run_time))

libs/apscheduler/jobstores/ram_store.py

@@ -8,7 +8,7 @@ from apscheduler.jobstores.base import JobStore
 class RAMJobStore(JobStore):
     def __init__(self):
         self.jobs = []
-    
+
     def add_job(self, job):
         self.jobs.append(job)
 

libs/apscheduler/jobstores/redis_store.py

@@ -0,0 +1,91 @@
+"""
+Stores jobs in a Redis database.
+"""
+from uuid import uuid4
+from datetime import datetime
+import logging
+
+from apscheduler.jobstores.base import JobStore
+from apscheduler.job import Job
+
+try:
+    import cPickle as pickle
+except ImportError:  # pragma: nocover
+    import pickle
+
+try:
+    from redis import StrictRedis
+except ImportError:  # pragma: nocover
+    raise ImportError('RedisJobStore requires redis installed')
+
+try:
+    long = long
+except NameError:
+    long = int
+
+logger = logging.getLogger(__name__)
+
+
+class RedisJobStore(JobStore):
+    def __init__(self, db=0, key_prefix='jobs.',
+                 pickle_protocol=pickle.HIGHEST_PROTOCOL, **connect_args):
+        self.jobs = []
+        self.pickle_protocol = pickle_protocol
+        self.key_prefix = key_prefix
+
+        if db is None:
+            raise ValueError('The "db" parameter must not be empty')
+        if not key_prefix:
+            raise ValueError('The "key_prefix" parameter must not be empty')
+
+        self.redis = StrictRedis(db=db, **connect_args)
+
+    def add_job(self, job):
+        job.id = str(uuid4())
+        job_state = job.__getstate__()
+        job_dict = {
+            'job_state': pickle.dumps(job_state, self.pickle_protocol),
+            'runs': '0',
+            'next_run_time': job_state.pop('next_run_time').isoformat()}
+        self.redis.hmset(self.key_prefix + job.id, job_dict)
+        self.jobs.append(job)
+
+    def remove_job(self, job):
+        self.redis.delete(self.key_prefix + job.id)
+        self.jobs.remove(job)
+
+    def load_jobs(self):
+        jobs = []
+        keys = self.redis.keys(self.key_prefix + '*')
+        pipeline = self.redis.pipeline()
+        for key in keys:
+            pipeline.hgetall(key)
+        results = pipeline.execute()
+
+        for job_dict in results:
+            job_state = {}
+            try:
+                job = Job.__new__(Job)
+                job_state = pickle.loads(job_dict['job_state'.encode()])
+                job_state['runs'] = long(job_dict['runs'.encode()])
+                dateval = job_dict['next_run_time'.encode()].decode()
+                job_state['next_run_time'] = datetime.strptime(
+                    dateval, '%Y-%m-%dT%H:%M:%S')
+                job.__setstate__(job_state)
+                jobs.append(job)
+            except Exception:
+                job_name = job_state.get('name', '(unknown)')
+                logger.exception('Unable to restore job "%s"', job_name)
+        self.jobs = jobs
+
+    def update_job(self, job):
+        attrs = {
+            'next_run_time': job.next_run_time.isoformat(),
+            'runs': job.runs}
+        self.redis.hmset(self.key_prefix + job.id, attrs)
+
+    def close(self):
+        self.redis.connection_pool.disconnect()
+
+    def __repr__(self):
+        return '<%s>' % self.__class__.__name__

libs/apscheduler/jobstores/shelve_store.py

@@ -32,17 +32,20 @@ class ShelveJobStore(JobStore):
 
     def add_job(self, job):
         job.id = self._generate_id()
-        self.jobs.append(job)
         self.store[job.id] = job.__getstate__()
+        self.store.sync()
+        self.jobs.append(job)
 
     def update_job(self, job):
         job_dict = self.store[job.id]
         job_dict['next_run_time'] = job.next_run_time
         job_dict['runs'] = job.runs
         self.store[job.id] = job_dict
+        self.store.sync()
 
     def remove_job(self, job):
         del self.store[job.id]
+        self.store.sync()
         self.jobs.remove(job)
 
     def load_jobs(self):

libs/apscheduler/jobstores/sqlalchemy_store.py

@@ -4,6 +4,8 @@ Stores jobs in a database table using SQLAlchemy.
 import pickle
 import logging
 
+import sqlalchemy
+
 from apscheduler.jobstores.base import JobStore
 from apscheduler.job import Job
 
@@ -28,17 +30,19 @@ class SQLAlchemyJobStore(JobStore):
         else:
             raise ValueError('Need either "engine" or "url" defined')
 
-        self.jobs_t = Table(tablename, metadata or MetaData(),
+        if sqlalchemy.__version__ < '0.7':
+            pickle_coltype = PickleType(pickle_protocol, mutable=False)
+        else:
+            pickle_coltype = PickleType(pickle_protocol)
+        self.jobs_t = Table(
+            tablename, metadata or MetaData(),
             Column('id', Integer,
                    Sequence(tablename + '_id_seq', optional=True),
                    primary_key=True),
-            Column('trigger', PickleType(pickle_protocol, mutable=False),
-                   nullable=False),
+            Column('trigger', pickle_coltype, nullable=False),
             Column('func_ref', String(1024), nullable=False),
-            Column('args', PickleType(pickle_protocol, mutable=False),
-                   nullable=False),
-            Column('kwargs', PickleType(pickle_protocol, mutable=False),
-                   nullable=False),
+            Column('args', pickle_coltype, nullable=False),
+            Column('kwargs', pickle_coltype, nullable=False),
             Column('name', Unicode(1024)),
             Column('misfire_grace_time', Integer, nullable=False),
             Column('coalesce', Boolean, nullable=False),

libs/apscheduler/scheduler.py

@@ -35,7 +35,7 @@ class Scheduler(object):
     their execution.
     """
 
-    _stopped = False
+    _stopped = True
     _thread = None
 
     def __init__(self, gconfig={}, **options):
@@ -60,6 +60,7 @@ class Scheduler(object):
         self.misfire_grace_time = int(config.pop('misfire_grace_time', 1))
         self.coalesce = asbool(config.pop('coalesce', True))
         self.daemonic = asbool(config.pop('daemonic', True))
+        self.standalone = asbool(config.pop('standalone', False))
 
         # Configure the thread pool
         if 'threadpool' in config:
@@ -85,6 +86,12 @@ class Scheduler(object):
     def start(self):
         """
         Starts the scheduler in a new thread.
+
+        In threaded mode (the default), this method will return immediately
+        after starting the scheduler thread.
+
+        In standalone mode, this method will block until there are no more
+        scheduled jobs.
         """
         if self.running:
             raise SchedulerAlreadyRunningError
@@ -99,11 +106,15 @@ class Scheduler(object):
         del self._pending_jobs[:]
 
         self._stopped = False
-        self._thread = Thread(target=self._main_loop, name='APScheduler')
-        self._thread.setDaemon(self.daemonic)
-        self._thread.start()
+        if self.standalone:
+            self._main_loop()
+        else:
+            self._thread = Thread(target=self._main_loop, name='APScheduler')
+            self._thread.setDaemon(self.daemonic)
+            self._thread.start()
 
-    def shutdown(self, wait=True, shutdown_threadpool=True):
+    def shutdown(self, wait=True, shutdown_threadpool=True,
+                 close_jobstores=True):
         """
         Shuts down the scheduler and terminates the thread.
         Does not interrupt any currently running jobs.
@@ -111,6 +122,7 @@ class Scheduler(object):
         :param wait: ``True`` to wait until all currently executing jobs have
                      finished (if ``shutdown_threadpool`` is also ``True``)
         :param shutdown_threadpool: ``True`` to shut down the thread pool
+        :param close_jobstores: ``True`` to close all job stores after shutdown
         """
         if not self.running:
             return
@@ -123,11 +135,19 @@ class Scheduler(object):
             self._threadpool.shutdown(wait)
 
         # Wait until the scheduler thread terminates
-        self._thread.join()
+        if self._thread:
+            self._thread.join()
+
+        # Close all job stores
+        if close_jobstores:
+            for jobstore in itervalues(self._jobstores):
+                jobstore.close()
 
     @property
     def running(self):
-        return not self._stopped and self._thread and self._thread.isAlive()
+        thread_alive = self._thread and self._thread.isAlive()
+        standalone = getattr(self, 'standalone', False)
+        return not self._stopped and (standalone or thread_alive)
 
     def add_jobstore(self, jobstore, alias, quiet=False):
         """
@@ -156,21 +176,25 @@ class Scheduler(object):
         if not quiet:
             self._wakeup.set()
 
-    def remove_jobstore(self, alias):
+    def remove_jobstore(self, alias, close=True):
         """
         Removes the job store by the given alias from this scheduler.
 
+        :param close: ``True`` to close the job store after removing it
         :type alias: str
         """
         self._jobstores_lock.acquire()
         try:
-            try:
-                del self._jobstores[alias]
-            except KeyError:
+            jobstore = self._jobstores.pop(alias)
+            if not jobstore:
                 raise KeyError('No such job store: %s' % alias)
         finally:
             self._jobstores_lock.release()
 
+        # Close the job store if requested
+        if close:
+            jobstore.close()
+
         # Notify listeners that a job store has been removed
         self._notify_listeners(JobStoreEvent(EVENT_JOBSTORE_REMOVED, alias))
 
@@ -245,8 +269,10 @@ class Scheduler(object):
                 **options):
         """
         Adds the given job to the job list and notifies the scheduler thread.
+        Any extra keyword arguments are passed along to the constructor of the
+        :class:`~apscheduler.job.Job` class (see :ref:`job_options`).
 
-        :param trigger: alias of the job store to store the job in
+        :param trigger: trigger that determines when ``func`` is called
         :param func: callable to run at the given time
         :param args: list of positional arguments to call func with
         :param kwargs: dict of keyword arguments to call func with
@@ -276,6 +302,8 @@ class Scheduler(object):
     def add_date_job(self, func, date, args=None, kwargs=None, **options):
         """
         Schedules a job to be completed on a specific date and time.
+        Any extra keyword arguments are passed along to the constructor of the
+        :class:`~apscheduler.job.Job` class (see :ref:`job_options`).
 
         :param func: callable to run at the given time
         :param date: the date/time to run the job at
@@ -294,6 +322,8 @@ class Scheduler(object):
                          **options):
         """
         Schedules a job to be completed on specified intervals.
+        Any extra keyword arguments are passed along to the constructor of the
+        :class:`~apscheduler.job.Job` class (see :ref:`job_options`).
 
         :param func: callable to run
         :param weeks: number of weeks to wait
@@ -322,6 +352,8 @@ class Scheduler(object):
         """
         Schedules a job to be completed on times that match the given
         expressions.
+        Any extra keyword arguments are passed along to the constructor of the
+        :class:`~apscheduler.job.Job` class (see :ref:`job_options`).
 
         :param func: callable to run
         :param year: year to run on
@@ -352,6 +384,8 @@ class Scheduler(object):
         This decorator does not wrap its host function.
         Unscheduling decorated functions is possible by passing the ``job``
         attribute of the scheduled function to :meth:`unschedule_job`.
+        Any extra keyword arguments are passed along to the constructor of the
+        :class:`~apscheduler.job.Job` class (see :ref:`job_options`).
         """
         def inner(func):
             func.job = self.add_cron_job(func, **options)
@@ -364,6 +398,8 @@ class Scheduler(object):
         This decorator does not wrap its host function.
         Unscheduling decorated functions is possible by passing the ``job``
         attribute of the scheduled function to :meth:`unschedule_job`.
+        Any extra keyword arguments are passed along to the constructor of the
+        :class:`~apscheduler.job.Job` class (see :ref:`job_options`).
         """
         def inner(func):
             func.job = self.add_interval_job(func, **options)
@@ -517,7 +553,8 @@ class Scheduler(object):
                             job.runs += len(run_times)
 
                         # Update the job, but don't keep finished jobs around
-                        if job.compute_next_run_time(now + timedelta(microseconds=1)):
+                        if job.compute_next_run_time(
+                                now + timedelta(microseconds=1)):
                             jobstore.update_job(job)
                         else:
                             self._remove_job(job, alias, jobstore)
@@ -550,10 +587,15 @@ class Scheduler(object):
                 logger.debug('Next wakeup is due at %s (in %f seconds)',
                              next_wakeup_time, wait_seconds)
                 self._wakeup.wait(wait_seconds)
+                self._wakeup.clear()
+            elif self.standalone:
+                logger.debug('No jobs left; shutting down scheduler')
+                self.shutdown()
+                break
             else:
                 logger.debug('No jobs; waiting until a job is added')
                 self._wakeup.wait()
-            self._wakeup.clear()
+                self._wakeup.clear()
 
         logger.info('Scheduler has been shut down')
         self._notify_listeners(SchedulerEvent(EVENT_SCHEDULER_SHUTDOWN))

libs/apscheduler/triggers/cron/__init__.py

@@ -21,8 +21,10 @@ class CronTrigger(object):
         if self.start_date:
             self.start_date = convert_to_datetime(self.start_date)
 
-        # Yank out all None valued fields
+        # Check field names and yank out all None valued fields
         for key, value in list(iteritems(values)):
+            if key not in self.FIELD_NAMES:
+                raise TypeError('Invalid field name: %s' % key)
             if value is None:
                 del values[key]
 
@@ -111,17 +113,17 @@ class CronTrigger(object):
 
             if next_value is None:
                 # No valid value was found
-                next_date, fieldnum = self._increment_field_value(next_date,
-                                                                  fieldnum - 1)
+                next_date, fieldnum = self._increment_field_value(
+                    next_date, fieldnum - 1)
             elif next_value > curr_value:
                 # A valid, but higher than the starting value, was found
                 if field.REAL:
-                    next_date = self._set_field_value(next_date, fieldnum,
-                                                      next_value)
+                    next_date = self._set_field_value(
+                        next_date, fieldnum, next_value)
                     fieldnum += 1
                 else:
-                    next_date, fieldnum = self._increment_field_value(next_date,
-                                                                      fieldnum)
+                    next_date, fieldnum = self._increment_field_value(
+                        next_date, fieldnum)
             else:
                 # A valid value was found, no changes necessary
                 fieldnum += 1

libs/apscheduler/triggers/cron/expressions.py

@@ -8,7 +8,7 @@ import re
 from apscheduler.util import asint
 
 __all__ = ('AllExpression', 'RangeExpression', 'WeekdayRangeExpression',
-           'WeekdayPositionExpression')
+           'WeekdayPositionExpression', 'LastDayOfMonthExpression')
 
 
 WEEKDAYS = ['mon', 'tue', 'wed', 'thu', 'fri', 'sat', 'sun']
@@ -176,3 +176,19 @@ class WeekdayPositionExpression(AllExpression):
         return "%s('%s', '%s')" % (self.__class__.__name__,
                                    self.options[self.option_num],
                                    WEEKDAYS[self.weekday])
+
+
+class LastDayOfMonthExpression(AllExpression):
+    value_re = re.compile(r'last', re.IGNORECASE)
+
+    def __init__(self):
+        pass
+
+    def get_next_value(self, date, field):
+        return monthrange(date.year, date.month)[1]
+
+    def __str__(self):
+        return 'last'
+
+    def __repr__(self):
+        return "%s()" % self.__class__.__name__

libs/apscheduler/triggers/cron/fields.py

@@ -85,7 +85,8 @@ class WeekField(BaseField):
 
 
 class DayOfMonthField(BaseField):
-    COMPILERS = BaseField.COMPILERS + [WeekdayPositionExpression]
+    COMPILERS = BaseField.COMPILERS + [WeekdayPositionExpression,
+                                       LastDayOfMonthExpression]
 
     def get_max(self, dateval):
         return monthrange(dateval.year, dateval.month)[1]

libs/apscheduler/util.py

@@ -6,7 +6,6 @@ from datetime import date, datetime, timedelta
 from time import mktime
 import re
 import sys
-from types import MethodType
 
 __all__ = ('asint', 'asbool', 'convert_to_datetime', 'timedelta_seconds',
            'time_difference', 'datetime_ceil', 'combine_opts',
@@ -64,7 +63,7 @@ def convert_to_datetime(input):
         return input
     elif isinstance(input, date):
         return datetime.fromordinal(input.toordinal())
-    elif isinstance(input, str):
+    elif isinstance(input, basestring):
         m = _DATE_REGEX.match(input)
         if not m:
             raise ValueError('Invalid date string')
@@ -109,7 +108,7 @@ def datetime_ceil(dateval):
     """
     if dateval.microsecond > 0:
         return dateval + timedelta(seconds=1,
-                                   microseconds= -dateval.microsecond)
+                                   microseconds=-dateval.microsecond)
     return dateval
 
 
@@ -143,7 +142,8 @@ def get_callable_name(func):
     if f_self and hasattr(func, '__name__'):
         if isinstance(f_self, type):
             # class method
-            return '%s.%s' % (f_self.__name__, func.__name__)
+            clsname = getattr(f_self, '__qualname__', None) or f_self.__name__
+            return '%s.%s' % (clsname, func.__name__)
         # bound method
         return '%s.%s' % (f_self.__class__.__name__, func.__name__)
 
@@ -169,7 +169,7 @@ def obj_to_ref(obj):
             raise ValueError
     except Exception:
         raise ValueError('Cannot determine the reference to %s' % repr(obj))
-    
+
     return ref
 
 

libs/backports/__init__.py

@@ -0,0 +1,3 @@
+# This is a Python "namespace package" http://www.python.org/dev/peps/pep-0382/
+from pkgutil import extend_path
+__path__ = extend_path(__path__, __name__)

libs/backports/ssl_match_hostname/README.txt

@@ -0,0 +1,42 @@
+
+The ssl.match_hostname() function from Python 3.2
+=================================================
+
+The Secure Sockets layer is only actually *secure*
+if you check the hostname in the certificate returned
+by the server to which you are connecting,
+and verify that it matches to hostname
+that you are trying to reach.
+
+But the matching logic, defined in `RFC2818`_,
+can be a bit tricky to implement on your own.
+So the ``ssl`` package in the Standard Library of Python 3.2
+now includes a ``match_hostname()`` function
+for performing this check instead of requiring every application
+to implement the check separately.
+
+This backport brings ``match_hostname()`` to users
+of earlier versions of Python.
+Simply make this distribution a dependency of your package,
+and then use it like this::
+
+    from backports.ssl_match_hostname import match_hostname, CertificateError
+    ...
+    sslsock = ssl.wrap_socket(sock, ssl_version=ssl.PROTOCOL_SSLv3,
+                              cert_reqs=ssl.CERT_REQUIRED, ca_certs=...)
+    try:
+        match_hostname(sslsock.getpeercert(), hostname)
+    except CertificateError, ce:
+        ...
+
+Note that the ``ssl`` module is only included in the Standard Library
+for Python 2.6 and later;
+users of Python 2.5 or earlier versions
+will also need to install the ``ssl`` distribution
+from the Python Package Index to use code like that shown above.
+
+Brandon Craig Rhodes is merely the packager of this distribution;
+the actual code inside comes verbatim from Python 3.2.
+
+.. _RFC2818: http://tools.ietf.org/html/rfc2818.html
+

libs/backports/ssl_match_hostname/__init__.py

@@ -0,0 +1,60 @@
+"""The match_hostname() function from Python 3.2, essential when using SSL."""
+
+import re
+
+__version__ = '3.2a3'
+
+class CertificateError(ValueError):
+    pass
+
+def _dnsname_to_pat(dn):
+    pats = []
+    for frag in dn.split(r'.'):
+        if frag == '*':
+            # When '*' is a fragment by itself, it matches a non-empty dotless
+            # fragment.
+            pats.append('[^.]+')
+        else:
+            # Otherwise, '*' matches any dotless fragment.
+            frag = re.escape(frag)
+            pats.append(frag.replace(r'\*', '[^.]*'))
+    return re.compile(r'\A' + r'\.'.join(pats) + r'\Z', re.IGNORECASE)
+
+def match_hostname(cert, hostname):
+    """Verify that *cert* (in decoded format as returned by
+    SSLSocket.getpeercert()) matches the *hostname*.  RFC 2818 rules
+    are mostly followed, but IP addresses are not accepted for *hostname*.
+
+    CertificateError is raised on failure. On success, the function
+    returns nothing.
+    """
+    if not cert:
+        raise ValueError("empty or no certificate")
+    dnsnames = []
+    san = cert.get('subjectAltName', ())
+    for key, value in san:
+        if key == 'DNS':
+            if _dnsname_to_pat(value).match(hostname):
+                return
+            dnsnames.append(value)
+    if not san:
+        # The subject is only checked when subjectAltName is empty
+        for sub in cert.get('subject', ()):
+            for key, value in sub:
+                # XXX according to RFC 2818, the most specific Common Name
+                # must be used.
+                if key == 'commonName':
+                    if _dnsname_to_pat(value).match(hostname):
+                        return
+                    dnsnames.append(value)
+    if len(dnsnames) > 1:
+        raise CertificateError("hostname %r "
+            "doesn't match either of %s"
+            % (hostname, ', '.join(map(repr, dnsnames))))
+    elif len(dnsnames) == 1:
+        raise CertificateError("hostname %r "
+            "doesn't match %r"
+            % (hostname, dnsnames[0]))
+    else:
+        raise CertificateError("no appropriate commonName or "
+            "subjectAltName fields were found")

libs/bencode/LICENSE.txt

@@ -1,143 +0,0 @@
-BitTorrent Open Source License
-
-Version 1.1
-
-This BitTorrent Open Source License (the "License") applies to the BitTorrent client and related software products as well as any updates or maintenance releases of that software ("BitTorrent Products") that are distributed by BitTorrent, Inc. ("Licensor").  Any BitTorrent Product licensed pursuant to this License is a Licensed Product. Licensed Product, in its entirety, is protected by U.S. copyright law.  This License identifies the terms under which you may use, copy, distribute or modify Licensed Product. 
-
-Preamble
-
-This Preamble is intended to describe, in plain English, the nature and scope of this License.  However, this Preamble is not a part of this license.  The legal effect of this License is dependent only upon the terms of the License and not this Preamble.
-
-This License complies with the Open Source Definition and is derived from the Jabber Open Source License 1.0 (the "JOSL"), which has been approved by Open Source Initiative. Sections 4(c) and 4(f)(iii) from the JOSL have been deleted.
-
-This License provides that:
-
-1. You may use or give away the Licensed Product, alone or as a component of an aggregate software distribution containing programs from several different sources.  No royalty or other fee is required.
-
-2. Both Source Code and executable versions of the Licensed Product, including Modifications made by previous Contributors, are available for your use.  (The terms "Licensed Product," "Modifications," "Contributors" and "Source Code" are defined in the License.)
-
-3. You are allowed to make Modifications to the Licensed Product, and you can create Derivative Works from it. (The term "Derivative Works" is defined in the License.)
-
-4.  By accepting the Licensed Product under the provisions of this License, you agree that any Modifications you make to the Licensed Product and then distribute are governed by the provisions of this License.  In particular, you must make the Source Code of your Modifications available to others free of charge and without a royalty.
-
-5.  You may sell, accept donations or otherwise receive compensation for executable versions of a Licensed Product, without paying a royalty or other fee to the Licensor or any Contributor, provided that such executable versions contain your or another Contributor?s material Modifications. For the avoidance of doubt, to the extent your executable version of a Licensed Product does not contain your or another Contributor?s material Modifications, you may not sell, accept donations or otherwise receive compensation for such executable.
-
-You may use the Licensed Product for any purpose, but the Licensor is not providing you any warranty whatsoever, nor is the Licensor accepting any liability in the event that the Licensed Product doesn't work properly or causes you any injury or damages.
-
-6.      If you sublicense the Licensed Product or Derivative Works, you may charge fees for warranty or support, or for accepting indemnity or liability obligations to your customers.  You cannot charge for, sell, accept donations or otherwise receive compensation for the Source Code.
-
-7.      If you assert any patent claims against the Licensor relating to the Licensed Product, or if you breach any terms of the License, your rights to the Licensed Product under this License automatically terminate.
-
-You may use this License to distribute your own Derivative Works, in which case the provisions of this License will apply to your Derivative Works just as they do to the original Licensed Product.
-
-Alternatively, you may distribute your Derivative Works under any other OSI-approved Open Source license, or under a proprietary license of your choice.  If you use any license other than this License, however, you must continue to fulfill the requirements of this License (including the provisions relating to publishing the Source Code) for those portions of your Derivative Works that consist of the Licensed Product, including the files containing Modifications.
-
-New versions of this License may be published from time to time in connection with new versions of a Licensed Product or otherwise.  You may choose to continue to use the license terms in this version of the License for the Licensed Product that was originally licensed hereunder, however, the new versions of this License will at all times apply to new versions of the Licensed Product released by Licensor after the release of the new version of this License.  Only the Licensor has the right to change the License terms as they apply to the Licensed Product. 
-
-This License relies on precise definitions for certain terms.  Those terms are defined when they are first used, and the definitions are repeated for your convenience in a Glossary at the end of the License.
-
-License Terms
-
-1.      Grant of License From Licensor.  Subject to the terms and conditions of this License, Licensor hereby grants you a world-wide, royalty-free, non-exclusive license, subject to third party intellectual property claims, to do the following:
-
-a.       Use, reproduce, modify, display, perform, sublicense and distribute any Modifications created by a Contributor or portions thereof, in both Source Code or as an executable program, either on an unmodified basis or as part of Derivative Works.
-
-b.       Under claims of patents now or hereafter owned or controlled by Contributor, to make, use, sell, offer for sale, have made, and/or otherwise dispose of Modifications or portions thereof, but solely to the extent that any such claim is necessary to enable you to make, use, sell, offer for sale, have made, and/or otherwise dispose of Modifications or portions thereof or Derivative Works thereof.
-
-2. Grant of License to Modifications From Contributor. "Modifications" means any additions to or deletions from the substance or structure of (i) a file containing a Licensed Product, or (ii) any new file that contains any part of a Licensed Product. Hereinafter in this License, the term "Licensed Product" shall include all previous Modifications that you receive from any Contributor.  Subject to the terms and conditions of this License, By application of the provisions in Section 4(a) below, each person or entity who created or contributed to the creation of, and distributed, a Modification (a "Contributor") hereby grants you a world-wide, royalty-free, non-exclusive license, subject to third party intellectual property claims, to do the following:
-
-a. Use, reproduce, modify, display, perform, sublicense and distribute any Modifications created by such Contributor or portions thereof, in both Source Code or as an executable program, either on an unmodified basis or as part of Derivative Works.
-
-b. Under claims of patents now or hereafter owned or controlled by Contributor, to make, use, sell, offer for sale, have made, and/or otherwise dispose of Modifications or portions thereof, but solely to the extent that any such claim is necessary to enable you to make, use, sell, offer for sale, have made, and/or otherwise dispose of Modifications or portions thereof or Derivative Works thereof. 
-
-3.      Exclusions From License Grant.  Nothing in this License shall be deemed to grant any rights to trademarks, copyrights, patents, trade secrets or any other intellectual property of Licensor or any Contributor except as expressly stated herein. No patent license is granted separate from the Licensed Product, for code that you delete from the Licensed Product, or for combinations of the Licensed Product with other software or hardware.  No right is granted to the trademarks of Licensor or any Contributor even if such marks are included in the Licensed Product. Nothing in this License shall be interpreted to prohibit Licensor from licensing under different terms from this License any code that Licensor otherwise would have a right to license.  As an express condition for your use of the Licensed Product, you hereby agree that you will not, without the prior written consent of Licensor, use any trademarks, copyrights, patents, trade secrets or any other intellectual property of Licensor or any Contributor except as expressly stated herein.  For the avoidance of doubt and without limiting the foregoing, you hereby agree that you will not use or display any trademark of Licensor or any Contributor in any domain name, directory filepath, advertisement, link or other reference to you in any manner or in any media.
-
-4.      Your Obligations Regarding Distribution. 
-
-a.       Application of This License to Your Modifications.  As an express condition for your use of the Licensed Product, you hereby agree that any Modifications that you create or to which you contribute, and which you distribute, are governed by the terms of this License including, without limitation, Section 2.  Any Modifications that you create or to which you contribute may be distributed only under the terms of this License or a future version of this License released under Section 7.  You must include a copy of this License with every copy of the Modifications you distribute.  You agree not to offer or impose any terms on any Source Code or executable version of the Licensed Product or Modifications that alter or restrict the applicable version of this License or the recipients' rights hereunder. However, you may include an additional document offering the additional rights described in Section 4(d).
-
-b.       Availability of Source Code.  You must make available, without charge, under the terms of this License, the Source Code of the Licensed Product and any Modifications that you distribute, either on the same media as you distribute any executable or other form of the Licensed Product, or via a mechanism generally accepted in the software development community for the electronic transfer of data (an "Electronic Distribution Mechanism").  The Source Code for any version of Licensed Product or Modifications that you distribute must remain available for as long as any executable or other form of the Licensed Product is distributed by you.  You are responsible for ensuring that the Source Code version remains available even if the Electronic Distribution Mechanism is maintained by a third party.
-
-c.       Intellectual Property Matters.  
-
-                                i.            Third Party Claims.  If you have knowledge that a license to a third party's intellectual property right is required to exercise the rights granted by this License, you must include a text file with the Source Code distribution titled "LEGAL" that describes the claim and the party making the claim in sufficient detail that a recipient will know whom to contact.  If you obtain such knowledge after you make any Modifications available as described in Section 4(b), you shall promptly modify the LEGAL file in all copies you make available thereafter and shall take other steps (such as notifying appropriate mailing lists or newsgroups) reasonably calculated to inform those who received the Licensed Product from you that new knowledge has been obtained.
-
-                               ii.            Contributor APIs.  If your Modifications include an application programming interface ("API") and you have knowledge of patent licenses that are reasonably necessary to implement that API, you must also include this information in the LEGAL file.
-
-                              iii.            Representations.  You represent that, except as disclosed pursuant to 4(c)(i) above, you believe that any Modifications you distribute are your original creations and that you have sufficient rights to grant the rights conveyed by this License.
-
-d.       Required Notices.  You must duplicate this License in any documentation you provide along with the Source Code of any Modifications you create or to which you contribute, and which you distribute, wherever you describe recipients' rights relating to Licensed Product.  You must duplicate the notice contained in Exhibit A (the "Notice") in each file of the Source Code of any copy you distribute of the Licensed Product. If you created a Modification, you may add your name as a Contributor to the Notice.  If it is not possible to put the Notice in a particular Source Code file due to its structure, then you must include such Notice in a location (such as a relevant directory file) where a user would be likely to look for such a notice.  You may choose to offer, and charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Licensed Product. However, you may do so only on your own behalf, and not on behalf of the Licensor or any Contributor.  You must make it clear that any such warranty, support, indemnity or liability obligation is offered by you alone, and you hereby agree to indemnify the Licensor and every Contributor for any liability incurred by the Licensor or such Contributor as a result of warranty, support, indemnity or liability terms you offer.
-
-e.        Distribution of Executable Versions.  You may distribute Licensed Product as an executable program under a license of your choice that may contain terms different from this License provided (i) you have satisfied the requirements of Sections 4(a) through 4(d) for that distribution, (ii) you include a conspicuous notice in the executable version, related documentation and collateral materials stating that the Source Code version of the
-Licensed Product is available under the terms of this License, including a description of how and where you have fulfilled the obligations of Section 4(b), and (iii) you make it clear that any terms that differ from this License are offered by you alone, not by Licensor or any Contributor.  You hereby agree to indemnify the Licensor and every Contributor for any liability incurred by Licensor or such Contributor as a result of any terms you offer. 
-
-f.       Distribution of Derivative Works.  You may create Derivative Works (e.g., combinations of some or all of the Licensed Product with other code) and distribute the Derivative Works as products under any other license you select, with the proviso that the requirements of this License are fulfilled for those portions of the Derivative Works that consist of the Licensed Product or any Modifications thereto.
-
-g.	Compensation for Distribution of Executable Versions of Licensed Products, Modifications or Derivative Works.  Notwithstanding any provision of this License to the contrary, by distributing, selling, licensing, sublicensing or otherwise making available any Licensed Product, or Modification or Derivative Work thereof, you and Licensor hereby acknowledge and agree that you may sell, license or sublicense for a fee, accept donations or otherwise receive compensation for executable versions of a Licensed Product, without paying a royalty or other fee to the Licensor or any other Contributor, provided that such executable versions (i) contain your or another Contributor?s material Modifications, or (ii) are otherwise material Derivative Works.  For purposes of this License, an executable version of the Licensed Product will be deemed to contain a material Modification, or will otherwise be deemed a material Derivative Work, if (a) the Licensed Product is modified with your own or a third party?s software programs or other code, and/or the Licensed Product is combined with a number of your own or a third party?s software programs or code, respectively, and (b) such software programs or code add or contribute material value, functionality or features to the License Product.  For the avoidance of doubt, to the extent your executable version of a Licensed Product does not contain your or another Contributor?s material Modifications or is otherwise not a material Derivative Work, in each case as contemplated herein, you may not sell, license or sublicense for a fee, accept donations or otherwise receive compensation for such executable.  Additionally, without limitation of the foregoing and notwithstanding any provision of this License to the contrary, you cannot charge for, sell, license or sublicense for a fee, accept donations or otherwise receive compensation for the Source Code. 
-
-5.      Inability to Comply Due to Statute or Regulation.  If it is impossible for you to comply with any of the terms of this License with respect to some or all of the Licensed Product due to statute, judicial order, or regulation, then you must (i) comply with the terms of this License to the maximum extent possible, (ii) cite the statute or regulation that prohibits you from adhering to the License, and (iii) describe the limitations and the code they affect. Such description must be included in the LEGAL file described in Section 4(d), and must be included with all distributions of the Source Code.  Except to the extent prohibited by statute or regulation, such description must be sufficiently detailed for a recipient of ordinary skill at computer programming to be able to understand it. 
-
-6.      Application of This License.  This License applies to code to which Licensor or Contributor has attached the Notice in Exhibit A, which is incorporated herein by this reference.
-
-7.      Versions of This License.
-
-a.       New Versions.  Licensor may publish from time to time revised and/or new versions of the License. 
-
-b.       Effect of New Versions.  Once Licensed Product has been published under a particular version of the License, you may always continue to use it under the terms of that version, provided that any such license be in full force and effect at the time, and has not been revoked or otherwise terminated.  You may also choose to use such Licensed Product under the terms of any subsequent version (but not any prior version) of the License published by Licensor.  No one other than Licensor has the right to modify the terms applicable to Licensed Product created under this License.
-
-c.       Derivative Works of this License.  If you create or use a modified version of this License, which you may do only in order to apply it to software that is not already a Licensed Product under this License, you must rename your license so that it is not confusingly similar to this License, and must make it clear that your license contains terms that differ from this License.  In so naming your license, you may not use any trademark of Licensor or any Contributor.
-
-8.      Disclaimer of Warranty.  LICENSED PRODUCT IS PROVIDED UNDER THIS LICENSE ON AN AS IS BASIS, WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING, WITHOUT LIMITATION, WARRANTIES THAT THE LICENSED PRODUCT IS FREE OF DEFECTS, MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE OR NON-INFRINGING. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE LICENSED PRODUCT IS WITH YOU.  SHOULD LICENSED PRODUCT PROVE DEFECTIVE IN ANY RESPECT, YOU (AND NOT THE LICENSOR OR ANY OTHER CONTRIBUTOR) ASSUME THE COST OF ANY NECESSARY SERVICING, REPAIR OR CORRECTION.  THIS
-DISCLAIMER OF WARRANTY CONSTITUTES AN ESSENTIAL PART OF THIS LICENSE. NO USE OF LICENSED PRODUCT IS AUTHORIZED HEREUNDER EXCEPT UNDER THIS DISCLAIMER.
-
-9.      Termination. 
-
-a.       Automatic Termination Upon Breach.  This license and the rights granted hereunder will terminate automatically if you fail to comply with the terms herein and fail to cure such breach within ten (10) days of being notified of the breach by the Licensor.  For purposes of this provision, proof of delivery via email to the address listed in the ?WHOIS? database of the registrar for any website through which you distribute or market any Licensed Product, or to any alternate email address which you designate in writing to the Licensor, shall constitute sufficient notification. All sublicenses to the Licensed Product that are properly granted shall survive any termination of this license so long as they continue to complye with the terms of this License.  Provisions that, by their nature, must remain in effect beyond the termination of this License, shall survive.
-
-b.       Termination Upon Assertion of Patent Infringement.  If you initiate litigation by asserting a patent infringement claim (excluding declaratory judgment actions) against Licensor or a Contributor (Licensor or Contributor against whom you file such an action is referred to herein as Respondent) alleging that Licensed Product directly or indirectly infringes any patent, then any and all rights granted by such Respondent to you under Sections 1 or 2 of this License shall terminate prospectively upon sixty (60) days notice from Respondent (the "Notice Period") unless within that Notice Period you either agree in writing (i) to pay Respondent a mutually agreeable reasonably royalty for your past or future use of Licensed Product made by such Respondent, or (ii) withdraw your litigation claim with respect to Licensed Product against such Respondent.  If within said Notice Period a reasonable royalty and payment arrangement are not mutually agreed upon in writing by the parties or the litigation claim is not withdrawn, the rights granted by Licensor to you under Sections 1 and 2 automatically terminate at the expiration of said Notice Period.
-
-c.       Reasonable Value of This License.  If you assert a patent infringement claim against Respondent alleging that Licensed Product directly or indirectly infringes any patent where such claim is resolved (such as by license or settlement) prior to the initiation of patent infringement litigation, then the reasonable value of the licenses granted by said Respondent under Sections 1 and 2 shall be taken into account in determining the amount or value of any payment or license.
-
-d.       No Retroactive Effect of Termination.  In the event of termination under Sections 9(a) or 9(b) above, all end user license agreements (excluding licenses to distributors and resellers) that have been validly granted by you or any distributor hereunder prior to termination shall survive termination.
-
-10.  Limitation of Liability.  UNDER NO CIRCUMSTANCES AND UNDER NO LEGAL THEORY, WHETHER TORT (INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE, SHALL THE LICENSOR, ANY CONTRIBUTOR, OR ANY DISTRIBUTOR OF LICENSED PRODUCT, OR ANY SUPPLIER OF ANY OF SUCH PARTIES, BE LIABLE TO ANY PERSON FOR ANY INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY CHARACTER INCLUDING, WITHOUT LIMITATION, DAMAGES FOR LOSS OF GOODWILL, WORK STOPPAGE, COMPUTER FAILURE OR MALFUNCTION, OR ANY AND ALL OTHER COMMERCIAL DAMAGES OR LOSSES, EVEN IF SUCH PARTY SHALL HAVE BEEN INFORMED OF THE POSSIBILITY OF SUCH DAMAGES.  THIS LIMITATION OF LIABILITY SHALL NOT APPLY TO LIABILITY FOR DEATH OR PERSONAL INJURY RESULTING FROM SUCH PARTYS NEGLIGENCE TO THE EXTENT APPLICABLE LAW PROHIBITS SUCH LIMITATION.  SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION OR LIMITATION OF INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO THIS EXCLUSION AND LIMITATION MAY NOT APPLY TO YOU. 
-
-11.  Responsibility for Claims.  As between Licensor and Contributors, each party is responsible for claims and damages arising, directly or indirectly, out of its utilization of rights under this License.  You agree to work with Licensor and Contributors to distribute such responsibility on an equitable basis.  Nothing herein is intended or shall be deemed to constitute any admission of liability.
-
-12.  U.S. Government End Users.  The Licensed Product is a commercial item, as that term is defined in 48 C.F.R. 2.101 (Oct. 1995), consisting of commercial computer software and commercial computer software documentation, as such terms are used in 48 C.F.R. 12.212 (Sept. 1995).  Consistent with 48 C.F.R. 12.212 and 48 C.F.R. 227.7202-1 through 227.7202-4 (June 1995), all U.S. Government End Users acquire Licensed Product with only those rights set forth herein.
-
-13.  Miscellaneous.  This License represents the complete agreement concerning the subject matter hereof.  If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable.  This License shall be governed by California law provisions (except to the extent applicable law, if any, provides otherwise), excluding its conflict-of-law provisions.  You expressly agree that in any litigation relating to this license the losing party shall be responsible for costs including, without limitation, court costs and reasonable attorneys fees and expenses.  The application of the United Nations Convention on Contracts for the International Sale of Goods is expressly excluded.  Any law or regulation that provides that the language of a contract shall be construed against the drafter shall not apply to this License.
-
-14.  Definition of You in This License. You throughout this License, whether in upper or lower case, means an individual or a legal entity exercising rights under, and complying with all of the terms of, this License or a future version of this License issued under Section 7.  For legal entities, you includes any entity that controls, is controlled by, is under common control with, or affiliated with, you.  For purposes of this definition, control means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity.  You are responsible for advising any affiliated entity of the terms of this License, and that any rights or privileges derived from or obtained by way of this License are subject to the restrictions outlined herein.
-
-15.  Glossary.  All defined terms in this License that are used in more than one Section of this License are repeated here, in alphabetical order, for the convenience of the reader.  The Section of this License in which each defined term is first used is shown in parentheses. 
-
-Contributor:  Each person or entity who created or contributed to the creation of, and distributed, a Modification. (See Section 2)
-
-Derivative Works: That term as used in this License is defined under U.S. copyright law.  (See Section 1(b))
-
-License:  This BitTorrent Open Source License.  (See first paragraph of License)
-
-Licensed Product:  Any BitTorrent Product licensed pursuant to this License.  The term "Licensed Product" includes all previous Modifications from any Contributor that you receive.  (See first paragraph of License and Section 2)
-
-Licensor:  BitTorrent, Inc.  (See first paragraph of License)
-
-Modifications:  Any additions to or deletions from the substance or structure of (i) a file containing Licensed Product, or (ii) any new file that contains any part of Licensed Product.  (See Section 2)
-
-Notice:  The notice contained in Exhibit A.  (See Section 4(e))
-
-Source Code: The preferred form for making modifications to the Licensed Product, including all modules contained therein, plus any associated interface definition files, scripts used to control compilation and installation of an executable program, or a list of differential comparisons against the Source Code of the Licensed Product.  (See Section 1(a))
-
-You:  This term is defined in Section 14 of this License.
-
-
-EXHIBIT A
-
-The Notice below must appear in each file of the Source Code of any copy you distribute of the Licensed Product or any hereto.  Contributors to any Modifications may add their own copyright notices to identify their own contributions.
-
-License:
-
-The contents of this file are subject to the BitTorrent Open Source License Version 1.0 (the License).  You may not copy or use this file, in either source code or executable form, except in compliance with the License.  You may obtain a copy of the License at http://www.bittorrent.com/license/.
-
-Software distributed under the License is distributed on an AS IS basis, WITHOUT WARRANTY OF ANY KIND, either express or implied.  See the License for the specific language governing rights and limitations under the License.
-

libs/bencode/__init__.py

@@ -1 +1,131 @@
-from bencode import *
+# The contents of this file are subject to the BitTorrent Open Source License
+# Version 1.1 (the License).  You may not copy or use this file, in either
+# source code or executable form, except in compliance with the License.  You
+# may obtain a copy of the License at http://www.bittorrent.com/license/.
+#
+# Software distributed under the License is distributed on an AS IS basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied.  See the License
+# for the specific language governing rights and limitations under the
+# License.
+
+# Written by Petru Paler
+
+from BTL import BTFailure
+
+
+def decode_int(x, f):
+    f += 1
+    newf = x.index('e', f)
+    n = int(x[f:newf])
+    if x[f] == '-':
+        if x[f + 1] == '0':
+            raise ValueError
+    elif x[f] == '0' and newf != f+1:
+        raise ValueError
+    return (n, newf+1)
+
+def decode_string(x, f):
+    colon = x.index(':', f)
+    n = int(x[f:colon])
+    if x[f] == '0' and colon != f+1:
+        raise ValueError
+    colon += 1
+    return (x[colon:colon+n], colon+n)
+
+def decode_list(x, f):
+    r, f = [], f+1
+    while x[f] != 'e':
+        v, f = decode_func[x[f]](x, f)
+        r.append(v)
+    return (r, f + 1)
+
+def decode_dict(x, f):
+    r, f = {}, f+1
+    while x[f] != 'e':
+        k, f = decode_string(x, f)
+        r[k], f = decode_func[x[f]](x, f)
+    return (r, f + 1)
+
+decode_func = {}
+decode_func['l'] = decode_list
+decode_func['d'] = decode_dict
+decode_func['i'] = decode_int
+decode_func['0'] = decode_string
+decode_func['1'] = decode_string
+decode_func['2'] = decode_string
+decode_func['3'] = decode_string
+decode_func['4'] = decode_string
+decode_func['5'] = decode_string
+decode_func['6'] = decode_string
+decode_func['7'] = decode_string
+decode_func['8'] = decode_string
+decode_func['9'] = decode_string
+
+def bdecode(x):
+    try:
+        r, l = decode_func[x[0]](x, 0)
+    except (IndexError, KeyError, ValueError):
+        raise BTFailure("not a valid bencoded string")
+    if l != len(x):
+        raise BTFailure("invalid bencoded value (data after valid prefix)")
+    return r
+
+from types import StringType, IntType, LongType, DictType, ListType, TupleType
+
+
+class Bencached(object):
+
+    __slots__ = ['bencoded']
+
+    def __init__(self, s):
+        self.bencoded = s
+
+def encode_bencached(x,r):
+    r.append(x.bencoded)
+
+def encode_int(x, r):
+    r.extend(('i', str(x), 'e'))
+
+def encode_bool(x, r):
+    if x:
+        encode_int(1, r)
+    else:
+        encode_int(0, r)
+        
+def encode_string(x, r):
+    r.extend((str(len(x)), ':', x))
+
+def encode_list(x, r):
+    r.append('l')
+    for i in x:
+        encode_func[type(i)](i, r)
+    r.append('e')
+
+def encode_dict(x,r):
+    r.append('d')
+    ilist = x.items()
+    ilist.sort()
+    for k, v in ilist:
+        r.extend((str(len(k)), ':', k))
+        encode_func[type(v)](v, r)
+    r.append('e')
+
+encode_func = {}
+encode_func[Bencached] = encode_bencached
+encode_func[IntType] = encode_int
+encode_func[LongType] = encode_int
+encode_func[StringType] = encode_string
+encode_func[ListType] = encode_list
+encode_func[TupleType] = encode_list
+encode_func[DictType] = encode_dict
+
+try:
+    from types import BooleanType
+    encode_func[BooleanType] = encode_bool
+except ImportError:
+    pass
+
+def bencode(x):
+    r = []
+    encode_func[type(x)](x, r)
+    return ''.join(r)

libs/bencode/bencode.py

@@ -1,131 +0,0 @@
-# The contents of this file are subject to the BitTorrent Open Source License
-# Version 1.1 (the License).  You may not copy or use this file, in either
-# source code or executable form, except in compliance with the License.  You
-# may obtain a copy of the License at http://www.bittorrent.com/license/.
-#
-# Software distributed under the License is distributed on an AS IS basis,
-# WITHOUT WARRANTY OF ANY KIND, either express or implied.  See the License
-# for the specific language governing rights and limitations under the
-# License.
-
-# Written by Petru Paler
-
-from BTL import BTFailure
-
-
-def decode_int(x, f):
-    f += 1
-    newf = x.index('e', f)
-    n = int(x[f:newf])
-    if x[f] == '-':
-        if x[f + 1] == '0':
-            raise ValueError
-    elif x[f] == '0' and newf != f+1:
-        raise ValueError
-    return (n, newf+1)
-
-def decode_string(x, f):
-    colon = x.index(':', f)
-    n = int(x[f:colon])
-    if x[f] == '0' and colon != f+1:
-        raise ValueError
-    colon += 1
-    return (x[colon:colon+n], colon+n)
-
-def decode_list(x, f):
-    r, f = [], f+1
-    while x[f] != 'e':
-        v, f = decode_func[x[f]](x, f)
-        r.append(v)
-    return (r, f + 1)
-
-def decode_dict(x, f):
-    r, f = {}, f+1
-    while x[f] != 'e':
-        k, f = decode_string(x, f)
-        r[k], f = decode_func[x[f]](x, f)
-    return (r, f + 1)
-
-decode_func = {}
-decode_func['l'] = decode_list
-decode_func['d'] = decode_dict
-decode_func['i'] = decode_int
-decode_func['0'] = decode_string
-decode_func['1'] = decode_string
-decode_func['2'] = decode_string
-decode_func['3'] = decode_string
-decode_func['4'] = decode_string
-decode_func['5'] = decode_string
-decode_func['6'] = decode_string
-decode_func['7'] = decode_string
-decode_func['8'] = decode_string
-decode_func['9'] = decode_string
-
-def bdecode(x):
-    try:
-        r, l = decode_func[x[0]](x, 0)
-    except (IndexError, KeyError, ValueError):
-        raise BTFailure("not a valid bencoded string")
-    if l != len(x):
-        raise BTFailure("invalid bencoded value (data after valid prefix)")
-    return r
-
-from types import StringType, IntType, LongType, DictType, ListType, TupleType
-
-
-class Bencached(object):
-
-    __slots__ = ['bencoded']
-
-    def __init__(self, s):
-        self.bencoded = s
-
-def encode_bencached(x,r):
-    r.append(x.bencoded)
-
-def encode_int(x, r):
-    r.extend(('i', str(x), 'e'))
-
-def encode_bool(x, r):
-    if x:
-        encode_int(1, r)
-    else:
-        encode_int(0, r)
-        
-def encode_string(x, r):
-    r.extend((str(len(x)), ':', x))
-
-def encode_list(x, r):
-    r.append('l')
-    for i in x:
-        encode_func[type(i)](i, r)
-    r.append('e')
-
-def encode_dict(x,r):
-    r.append('d')
-    ilist = x.items()
-    ilist.sort()
-    for k, v in ilist:
-        r.extend((str(len(k)), ':', k))
-        encode_func[type(v)](v, r)
-    r.append('e')
-
-encode_func = {}
-encode_func[Bencached] = encode_bencached
-encode_func[IntType] = encode_int
-encode_func[LongType] = encode_int
-encode_func[StringType] = encode_string
-encode_func[ListType] = encode_list
-encode_func[TupleType] = encode_list
-encode_func[DictType] = encode_dict
-
-try:
-    from types import BooleanType
-    encode_func[BooleanType] = encode_bool
-except ImportError:
-    pass
-
-def bencode(x):
-    r = []
-    encode_func[type(x)](x, r)
-    return ''.join(r)

libs/bs4/__init__.py

@@ -17,16 +17,17 @@ http://www.crummy.com/software/BeautifulSoup/bs4/doc/
 """
 
 __author__ = "Leonard Richardson (leonardr@segfault.org)"
-__version__ = "4.1.0"
-__copyright__ = "Copyright (c) 2004-2012 Leonard Richardson"
+__version__ = "4.3.2"
+__copyright__ = "Copyright (c) 2004-2013 Leonard Richardson"
 __license__ = "MIT"
 
 __all__ = ['BeautifulSoup']
 
+import os
 import re
 import warnings
 
-from .builder import builder_registry
+from .builder import builder_registry, ParserRejectedMarkup
 from .dammit import UnicodeDammit
 from .element import (
     CData,
@@ -74,11 +75,7 @@ class BeautifulSoup(Tag):
     # want, look for one with these features.
     DEFAULT_BUILDER_FEATURES = ['html', 'fast']
 
-    # Used when determining whether a text node is all whitespace and
-    # can be replaced with a single space. A text node that contains
-    # fancy Unicode spaces (usually non-breaking) should be left
-    # alone.
-    STRIP_ASCII_SPACES = {9: None, 10: None, 12: None, 13: None, 32: None, }
+    ASCII_SPACES = '\x20\x0a\x09\x0c\x0d'
 
     def __init__(self, markup="", features=None, builder=None,
                  parse_only=None, from_encoding=None, **kwargs):
@@ -149,7 +146,7 @@ class BeautifulSoup(Tag):
                 features = self.DEFAULT_BUILDER_FEATURES
             builder_class = builder_registry.lookup(*features)
             if builder_class is None:
-                raise ValueError(
+                raise FeatureNotFound(
                     "Couldn't find a tree builder with the features you "
                     "requested: %s. Do you need to install a parser library?"
                     % ",".join(features))
@@ -160,18 +157,46 @@ class BeautifulSoup(Tag):
 
         self.parse_only = parse_only
 
-        self.reset()
-
         if hasattr(markup, 'read'):        # It's a file-type object.
             markup = markup.read()
-        (self.markup, self.original_encoding, self.declared_html_encoding,
-         self.contains_replacement_characters) = (
-            self.builder.prepare_markup(markup, from_encoding))
+        elif len(markup) <= 256:
+            # Print out warnings for a couple beginner problems
+            # involving passing non-markup to Beautiful Soup.
+            # Beautiful Soup will still parse the input as markup,
+            # just in case that's what the user really wants.
+            if (isinstance(markup, unicode)
+                and not os.path.supports_unicode_filenames):
+                possible_filename = markup.encode("utf8")
+            else:
+                possible_filename = markup
+            is_file = False
+            try:
+                is_file = os.path.exists(possible_filename)
+            except Exception, e:
+                # This is almost certainly a problem involving
+                # characters not valid in filenames on this
+                # system. Just let it go.
+                pass
+            if is_file:
+                warnings.warn(
+                    '"%s" looks like a filename, not markup. You should probably open this file and pass the filehandle into Beautiful Soup.' % markup)
+            if markup[:5] == "http:" or markup[:6] == "https:":
+                # TODO: This is ugly but I couldn't get it to work in
+                # Python 3 otherwise.
+                if ((isinstance(markup, bytes) and not b' ' in markup)
+                    or (isinstance(markup, unicode) and not u' ' in markup)):
+                    warnings.warn(
+                        '"%s" looks like a URL. Beautiful Soup is not an HTTP client. You should probably use an HTTP client to get the document behind the URL, and feed that document to Beautiful Soup.' % markup)
 
-        try:
-            self._feed()
-        except StopParsing:
-            pass
+        for (self.markup, self.original_encoding, self.declared_html_encoding,
+         self.contains_replacement_characters) in (
+            self.builder.prepare_markup(markup, from_encoding)):
+            self.reset()
+            try:
+                self._feed()
+                break
+            except ParserRejectedMarkup:
+                pass
 
         # Clear out the markup and remove the builder's circular
         # reference to this object.
@@ -192,29 +217,32 @@ class BeautifulSoup(Tag):
         Tag.__init__(self, self, self.builder, self.ROOT_TAG_NAME)
         self.hidden = 1
         self.builder.reset()
-        self.currentData = []
+        self.current_data = []
         self.currentTag = None
         self.tagStack = []
+        self.preserve_whitespace_tag_stack = []
         self.pushTag(self)
 
     def new_tag(self, name, namespace=None, nsprefix=None, **attrs):
         """Create a new tag associated with this soup."""
         return Tag(None, self.builder, name, namespace, nsprefix, attrs)
 
-    def new_string(self, s):
+    def new_string(self, s, subclass=NavigableString):
         """Create a new NavigableString associated with this soup."""
-        navigable = NavigableString(s)
+        navigable = subclass(s)
         navigable.setup()
         return navigable
 
     def insert_before(self, successor):
-        raise ValueError("BeautifulSoup objects don't support insert_before().")
+        raise NotImplementedError("BeautifulSoup objects don't support insert_before().")
 
     def insert_after(self, successor):
-        raise ValueError("BeautifulSoup objects don't support insert_after().")
+        raise NotImplementedError("BeautifulSoup objects don't support insert_after().")
 
     def popTag(self):
         tag = self.tagStack.pop()
+        if self.preserve_whitespace_tag_stack and tag == self.preserve_whitespace_tag_stack[-1]:
+            self.preserve_whitespace_tag_stack.pop()
         #print "Pop", tag.name
         if self.tagStack:
             self.currentTag = self.tagStack[-1]
@@ -226,32 +254,49 @@ class BeautifulSoup(Tag):
             self.currentTag.contents.append(tag)
         self.tagStack.append(tag)
         self.currentTag = self.tagStack[-1]
+        if tag.name in self.builder.preserve_whitespace_tags:
+            self.preserve_whitespace_tag_stack.append(tag)
 
     def endData(self, containerClass=NavigableString):
-        if self.currentData:
-            currentData = u''.join(self.currentData)
-            if (currentData.translate(self.STRIP_ASCII_SPACES) == '' and
-                not set([tag.name for tag in self.tagStack]).intersection(
-                    self.builder.preserve_whitespace_tags)):
-                if '\n' in currentData:
-                    currentData = '\n'
-                else:
-                    currentData = ' '
-            self.currentData = []
+        if self.current_data:
+            current_data = u''.join(self.current_data)
+            # If whitespace is not preserved, and this string contains
+            # nothing but ASCII spaces, replace it with a single space
+            # or newline.
+            if not self.preserve_whitespace_tag_stack:
+                strippable = True
+                for i in current_data:
+                    if i not in self.ASCII_SPACES:
+                        strippable = False
+                        break
+                if strippable:
+                    if '\n' in current_data:
+                        current_data = '\n'
+                    else:
+                        current_data = ' '
+
+            # Reset the data collector.
+            self.current_data = []
+
+            # Should we add this string to the tree at all?
             if self.parse_only and len(self.tagStack) <= 1 and \
                    (not self.parse_only.text or \
-                    not self.parse_only.search(currentData)):
+                    not self.parse_only.search(current_data)):
                 return
-            o = containerClass(currentData)
+
+            o = containerClass(current_data)
             self.object_was_parsed(o)
 
-    def object_was_parsed(self, o):
+    def object_was_parsed(self, o, parent=None, most_recent_element=None):
         """Add an object to the parse tree."""
-        o.setup(self.currentTag, self.previous_element)
-        if self.previous_element:
-            self.previous_element.next_element = o
-        self.previous_element = o
-        self.currentTag.contents.append(o)
+        parent = parent or self.currentTag
+        most_recent_element = most_recent_element or self._most_recent_element
+        o.setup(parent, most_recent_element)
+
+        if most_recent_element is not None:
+            most_recent_element.next_element = o
+        self._most_recent_element = o
+        parent.contents.append(o)
 
     def _popToTag(self, name, nsprefix=None, inclusivePop=True):
         """Pops the tag stack up to and including the most recent
@@ -260,22 +305,21 @@ class BeautifulSoup(Tag):
         the given tag."""
         #print "Popping to %s" % name
         if name == self.ROOT_TAG_NAME:
+            # The BeautifulSoup object itself can never be popped.
             return
 
-        numPops = 0
-        mostRecentTag = None
+        most_recently_popped = None
 
-        for i in range(len(self.tagStack) - 1, 0, -1):
-            if (name == self.tagStack[i].name
-                and nsprefix == self.tagStack[i].nsprefix == nsprefix):
-                numPops = len(self.tagStack) - i
+        stack_size = len(self.tagStack)
+        for i in range(stack_size - 1, 0, -1):
+            t = self.tagStack[i]
+            if (name == t.name and nsprefix == t.prefix):
+                if inclusivePop:
+                    most_recently_popped = self.popTag()
                 break
-        if not inclusivePop:
-            numPops = numPops - 1
+            most_recently_popped = self.popTag()
 
-        for i in range(0, numPops):
-            mostRecentTag = self.popTag()
-        return mostRecentTag
+        return most_recently_popped
 
     def handle_starttag(self, name, namespace, nsprefix, attrs):
         """Push a start tag on to the stack.
@@ -295,12 +339,12 @@ class BeautifulSoup(Tag):
             return None
 
         tag = Tag(self, self.builder, name, namespace, nsprefix, attrs,
-                  self.currentTag, self.previous_element)
+                  self.currentTag, self._most_recent_element)
         if tag is None:
             return tag
-        if self.previous_element:
-            self.previous_element.next_element = tag
-        self.previous_element = tag
+        if self._most_recent_element:
+            self._most_recent_element.next_element = tag
+        self._most_recent_element = tag
         self.pushTag(tag)
         return tag
 
@@ -310,7 +354,7 @@ class BeautifulSoup(Tag):
         self._popToTag(name, nsprefix)
 
     def handle_data(self, data):
-        self.currentData.append(data)
+        self.current_data.append(data)
 
     def decode(self, pretty_print=False,
                eventual_encoding=DEFAULT_OUTPUT_ENCODING,
@@ -333,6 +377,10 @@ class BeautifulSoup(Tag):
         return prefix + super(BeautifulSoup, self).decode(
             indent_level, eventual_encoding, formatter)
 
+# Alias to make it easier to type import: 'from bs4 import _soup'
+_s = BeautifulSoup
+_soup = BeautifulSoup
+
 class BeautifulStoneSoup(BeautifulSoup):
     """Deprecated interface to an XML parser."""
 
@@ -347,6 +395,9 @@ class BeautifulStoneSoup(BeautifulSoup):
 class StopParsing(Exception):
     pass
 
+class FeatureNotFound(ValueError):
+    pass
+
 
 #By default, act as an HTML pretty-printer.
 if __name__ == '__main__':

libs/bs4/builder/__init__.py

@@ -147,18 +147,29 @@ class TreeBuilder(object):
 
         Modifies its input in place.
         """
+        if not attrs:
+            return attrs
         if self.cdata_list_attributes:
             universal = self.cdata_list_attributes.get('*', [])
             tag_specific = self.cdata_list_attributes.get(
-                tag_name.lower(), [])
-            for cdata_list_attr in itertools.chain(universal, tag_specific):
-                if cdata_list_attr in dict(attrs):
-                    # Basically, we have a "class" attribute whose
-                    # value is a whitespace-separated list of CSS
-                    # classes. Split it into a list.
-                    value = attrs[cdata_list_attr]
-                    values = whitespace_re.split(value)
-                    attrs[cdata_list_attr] = values
+                tag_name.lower(), None)
+            for attr in attrs.keys():
+                if attr in universal or (tag_specific and attr in tag_specific):
+                    # We have a "class"-type attribute whose string
+                    # value is a whitespace-separated list of
+                    # values. Split it into a list.
+                    value = attrs[attr]
+                    if isinstance(value, basestring):
+                        values = whitespace_re.split(value)
+                    else:
+                        # html5lib sometimes calls setAttributes twice
+                        # for the same tag when rearranging the parse
+                        # tree. On the second call the attribute value
+                        # here is already a list.  If this happens,
+                        # leave the value alone rather than trying to
+                        # split it again.
+                        values = value
+                    attrs[attr] = values
         return attrs
 
 class SAXTreeBuilder(TreeBuilder):
@@ -287,6 +298,9 @@ def register_treebuilders_from(module):
             # Register the builder while we're at it.
             this_module.builder_registry.register(obj)
 
+class ParserRejectedMarkup(Exception):
+    pass
+
 # Builders are registered in reverse order of priority, so that custom
 # builder registrations will take precedence. In general, we want lxml
 # to take precedence over html5lib, because it's faster. And we only

libs/bs4/builder/_html5lib.py

@@ -27,7 +27,7 @@ class HTML5TreeBuilder(HTMLTreeBuilder):
     def prepare_markup(self, markup, user_specified_encoding):
         # Store the user-specified encoding for use later on.
         self.user_specified_encoding = user_specified_encoding
-        return markup, None, None, False
+        yield (markup, None, None, False)
 
     # These methods are defined by Beautiful Soup.
     def feed(self, markup):
@@ -123,17 +123,50 @@ class Element(html5lib.treebuilders._base.Node):
         self.namespace = namespace
 
     def appendChild(self, node):
-        if (node.element.__class__ == NavigableString and self.element.contents
+        string_child = child = None
+        if isinstance(node, basestring):
+            # Some other piece of code decided to pass in a string
+            # instead of creating a TextElement object to contain the
+            # string.
+            string_child = child = node
+        elif isinstance(node, Tag):
+            # Some other piece of code decided to pass in a Tag
+            # instead of creating an Element object to contain the
+            # Tag.
+            child = node
+        elif node.element.__class__ == NavigableString:
+            string_child = child = node.element
+        else:
+            child = node.element
+
+        if not isinstance(child, basestring) and child.parent is not None:
+            node.element.extract()
+
+        if (string_child and self.element.contents
             and self.element.contents[-1].__class__ == NavigableString):
-            # Concatenate new text onto old text node
-            # XXX This has O(n^2) performance, for input like
+            # We are appending a string onto another string.
+            # TODO This has O(n^2) performance, for input like
             # "a</a>a</a>a</a>..."
             old_element = self.element.contents[-1]
-            new_element = self.soup.new_string(old_element + node.element)
+            new_element = self.soup.new_string(old_element + string_child)
             old_element.replace_with(new_element)
+            self.soup._most_recent_element = new_element
         else:
-            self.element.append(node.element)
-            node.parent = self
+            if isinstance(node, basestring):
+                # Create a brand new NavigableString from this string.
+                child = self.soup.new_string(node)
+
+            # Tell Beautiful Soup to act as if it parsed this element
+            # immediately after the parent's last descendant. (Or
+            # immediately after the parent, if it has no children.)
+            if self.element.contents:
+                most_recent_element = self.element._last_descendant(False)
+            else:
+                most_recent_element = self.element
+
+            self.soup.object_was_parsed(
+                child, parent=self.element,
+                most_recent_element=most_recent_element)
 
     def getAttributes(self):
         return AttrList(self.element)
@@ -162,11 +195,11 @@ class Element(html5lib.treebuilders._base.Node):
     attributes = property(getAttributes, setAttributes)
 
     def insertText(self, data, insertBefore=None):
-        text = TextNode(self.soup.new_string(data), self.soup)
         if insertBefore:
-            self.insertBefore(text, insertBefore)
+            text = TextNode(self.soup.new_string(data), self.soup)
+            self.insertBefore(data, insertBefore)
         else:
-            self.appendChild(text)
+            self.appendChild(data)
 
     def insertBefore(self, node, refNode):
         index = self.element.index(refNode.element)
@@ -183,16 +216,46 @@ class Element(html5lib.treebuilders._base.Node):
     def removeChild(self, node):
         node.element.extract()
 
-    def reparentChildren(self, newParent):
-        while self.element.contents:
-            child = self.element.contents[0]
-            child.extract()
-            if isinstance(child, Tag):
-                newParent.appendChild(
-                    Element(child, self.soup, namespaces["html"]))
-            else:
-                newParent.appendChild(
-                    TextNode(child, self.soup))
+    def reparentChildren(self, new_parent):
+        """Move all of this tag's children into another tag."""
+        element = self.element
+        new_parent_element = new_parent.element
+        # Determine what this tag's next_element will be once all the children
+        # are removed.
+        final_next_element = element.next_sibling
+
+        new_parents_last_descendant = new_parent_element._last_descendant(False, False)
+        if len(new_parent_element.contents) > 0:
+            # The new parent already contains children. We will be
+            # appending this tag's children to the end.
+            new_parents_last_child = new_parent_element.contents[-1]
+            new_parents_last_descendant_next_element = new_parents_last_descendant.next_element
+        else:
+            # The new parent contains no children.
+            new_parents_last_child = None
+            new_parents_last_descendant_next_element = new_parent_element.next_element
+
+        to_append = element.contents
+        append_after = new_parent.element.contents
+        if len(to_append) > 0:
+            # Set the first child's previous_element and previous_sibling
+            # to elements within the new parent
+            first_child = to_append[0]
+            first_child.previous_element = new_parents_last_descendant
+            first_child.previous_sibling = new_parents_last_child
+
+            # Fix the last child's next_element and next_sibling
+            last_child = to_append[-1]
+            last_child.next_element = new_parents_last_descendant_next_element
+            last_child.next_sibling = None
+
+        for child in to_append:
+            child.parent = new_parent_element
+            new_parent_element.contents.append(child)
+
+        # Now that this element has no children, change its .next_element.
+        element.contents = []
+        element.next_element = final_next_element
 
     def cloneNode(self):
         tag = self.soup.new_tag(self.element.name, self.namespace)

libs/bs4/builder/_htmlparser.py

@@ -45,7 +45,15 @@ HTMLPARSER = 'html.parser'
 class BeautifulSoupHTMLParser(HTMLParser):
     def handle_starttag(self, name, attrs):
         # XXX namespace
-        self.soup.handle_starttag(name, None, None, dict(attrs))
+        attr_dict = {}
+        for key, value in attrs:
+            # Change None attribute values to the empty string
+            # for consistency with the other tree builders.
+            if value is None:
+                value = ''
+            attr_dict[key] = value
+            attrvalue = '""'
+        self.soup.handle_starttag(name, None, None, attr_dict)
 
     def handle_endtag(self, name):
         self.soup.handle_endtag(name)
@@ -58,6 +66,8 @@ class BeautifulSoupHTMLParser(HTMLParser):
         # it's fixed.
         if name.startswith('x'):
             real_name = int(name.lstrip('x'), 16)
+        elif name.startswith('X'):
+            real_name = int(name.lstrip('X'), 16)
         else:
             real_name = int(name)
 
@@ -85,6 +95,9 @@ class BeautifulSoupHTMLParser(HTMLParser):
         self.soup.endData()
         if data.startswith("DOCTYPE "):
             data = data[len("DOCTYPE "):]
+        elif data == 'DOCTYPE':
+            # i.e. "<!DOCTYPE>"
+            data = ''
         self.soup.handle_data(data)
         self.soup.endData(Doctype)
 
@@ -130,13 +143,14 @@ class HTMLParserTreeBuilder(HTMLTreeBuilder):
         replaced with REPLACEMENT CHARACTER).
         """
         if isinstance(markup, unicode):
-            return markup, None, None, False
+            yield (markup, None, None, False)
+            return
 
         try_encodings = [user_specified_encoding, document_declared_encoding]
         dammit = UnicodeDammit(markup, try_encodings, is_html=True)
-        return (dammit.markup, dammit.original_encoding,
-                dammit.declared_html_encoding,
-                dammit.contains_replacement_characters)
+        yield (dammit.markup, dammit.original_encoding,
+               dammit.declared_html_encoding,
+               dammit.contains_replacement_characters)
 
     def feed(self, markup):
         args, kwargs = self.parser_args

libs/bs4/builder/_lxml.py

@@ -3,6 +3,7 @@ __all__ = [
     'LXMLTreeBuilder',
     ]
 
+from io import BytesIO
 from StringIO import StringIO
 import collections
 from lxml import etree
@@ -12,9 +13,10 @@ from bs4.builder import (
     HTML,
     HTMLTreeBuilder,
     PERMISSIVE,
+    ParserRejectedMarkup,
     TreeBuilder,
     XML)
-from bs4.dammit import UnicodeDammit
+from bs4.dammit import EncodingDetector
 
 LXML = 'lxml'
 
@@ -28,24 +30,36 @@ class LXMLTreeBuilderForXML(TreeBuilder):
 
     CHUNK_SIZE = 512
 
-    @property
-    def default_parser(self):
+    # This namespace mapping is specified in the XML Namespace
+    # standard.
+    DEFAULT_NSMAPS = {'http://www.w3.org/XML/1998/namespace' : "xml"}
+
+    def default_parser(self, encoding):
         # This can either return a parser object or a class, which
         # will be instantiated with default arguments.
-        return etree.XMLParser(target=self, strip_cdata=False, recover=True)
+        if self._default_parser is not None:
+            return self._default_parser
+        return etree.XMLParser(
+            target=self, strip_cdata=False, recover=True, encoding=encoding)
+
+    def parser_for(self, encoding):
+        # Use the default parser.
+        parser = self.default_parser(encoding)
 
-    def __init__(self, parser=None, empty_element_tags=None):
-        if empty_element_tags is not None:
-            self.empty_element_tags = set(empty_element_tags)
-        if parser is None:
-            # Use the default parser.
-            parser = self.default_parser
         if isinstance(parser, collections.Callable):
             # Instantiate the parser with default arguments
-            parser = parser(target=self, strip_cdata=False)
-        self.parser = parser
+            parser = parser(target=self, strip_cdata=False, encoding=encoding)
+        return parser
+
+    def __init__(self, parser=None, empty_element_tags=None):
+        # TODO: Issue a warning if parser is present but not a
+        # callable, since that means there's no way to create new
+        # parsers for different encodings.
+        self._default_parser = parser
+        if empty_element_tags is not None:
+            self.empty_element_tags = set(empty_element_tags)
         self.soup = None
-        self.nsmaps = None
+        self.nsmaps = [self.DEFAULT_NSMAPS]
 
     def _getNsTag(self, tag):
         # Split the namespace URL out of a fully-qualified lxml tag
@@ -58,50 +72,69 @@ class LXMLTreeBuilderForXML(TreeBuilder):
     def prepare_markup(self, markup, user_specified_encoding=None,
                        document_declared_encoding=None):
         """
-        :return: A 3-tuple (markup, original encoding, encoding
-        declared within markup).
+        :yield: A series of 4-tuples.
+         (markup, encoding, declared encoding,
+          has undergone character replacement)
+
+        Each 4-tuple represents a strategy for parsing the document.
         """
         if isinstance(markup, unicode):
-            return markup, None, None, False
+            # We were given Unicode. Maybe lxml can parse Unicode on
+            # this system?
+            yield markup, None, document_declared_encoding, False
 
+        if isinstance(markup, unicode):
+            # No, apparently not. Convert the Unicode to UTF-8 and
+            # tell lxml to parse it as UTF-8.
+            yield (markup.encode("utf8"), "utf8",
+                   document_declared_encoding, False)
+
+        # Instead of using UnicodeDammit to convert the bytestring to
+        # Unicode using different encodings, use EncodingDetector to
+        # iterate over the encodings, and tell lxml to try to parse
+        # the document as each one in turn.
+        is_html = not self.is_xml
         try_encodings = [user_specified_encoding, document_declared_encoding]
-        dammit = UnicodeDammit(markup, try_encodings, is_html=True)
-        return (dammit.markup, dammit.original_encoding,
-                dammit.declared_html_encoding,
-                dammit.contains_replacement_characters)
+        detector = EncodingDetector(markup, try_encodings, is_html)
+        for encoding in detector.encodings:
+            yield (detector.markup, encoding, document_declared_encoding, False)
 
     def feed(self, markup):
-        if isinstance(markup, basestring):
+        if isinstance(markup, bytes):
+            markup = BytesIO(markup)
+        elif isinstance(markup, unicode):
             markup = StringIO(markup)
+
         # Call feed() at least once, even if the markup is empty,
         # or the parser won't be initialized.
         data = markup.read(self.CHUNK_SIZE)
-        self.parser.feed(data)
-        while data != '':
-            # Now call feed() on the rest of the data, chunk by chunk.
-            data = markup.read(self.CHUNK_SIZE)
-            if data != '':
-                self.parser.feed(data)
-        self.parser.close()
+        try:
+            self.parser = self.parser_for(self.soup.original_encoding)
+            self.parser.feed(data)
+            while len(data) != 0:
+                # Now call feed() on the rest of the data, chunk by chunk.
+                data = markup.read(self.CHUNK_SIZE)
+                if len(data) != 0:
+                    self.parser.feed(data)
+            self.parser.close()
+        except (UnicodeDecodeError, LookupError, etree.ParserError), e:
+            raise ParserRejectedMarkup(str(e))
 
     def close(self):
-        self.nsmaps = None
+        self.nsmaps = [self.DEFAULT_NSMAPS]
 
     def start(self, name, attrs, nsmap={}):
         # Make sure attrs is a mutable dict--lxml may send an immutable dictproxy.
         attrs = dict(attrs)
-
         nsprefix = None
         # Invert each namespace map as it comes in.
-        if len(nsmap) == 0 and self.nsmaps != None:
-            # There are no new namespaces for this tag, but namespaces
-            # are in play, so we need a separate tag stack to know
-            # when they end.
+        if len(self.nsmaps) > 1:
+            # There are no new namespaces for this tag, but
+            # non-default namespaces are in play, so we need a
+            # separate tag stack to know when they end.
             self.nsmaps.append(None)
         elif len(nsmap) > 0:
             # A new namespace mapping has come into play.
-            if self.nsmaps is None:
-                self.nsmaps = []
             inverted_nsmap = dict((value, key) for key, value in nsmap.items())
             self.nsmaps.append(inverted_nsmap)
             # Also treat the namespace mapping as a set of attributes on the
@@ -111,14 +144,34 @@ class LXMLTreeBuilderForXML(TreeBuilder):
                 attribute = NamespacedAttribute(
                     "xmlns", prefix, "http://www.w3.org/2000/xmlns/")
                 attrs[attribute] = namespace
+
+        # Namespaces are in play. Find any attributes that came in
+        # from lxml with namespaces attached to their names, and
+        # turn then into NamespacedAttribute objects.
+        new_attrs = {}
+        for attr, value in attrs.items():
+            namespace, attr = self._getNsTag(attr)
+            if namespace is None:
+                new_attrs[attr] = value
+            else:
+                nsprefix = self._prefix_for_namespace(namespace)
+                attr = NamespacedAttribute(nsprefix, attr, namespace)
+                new_attrs[attr] = value
+        attrs = new_attrs
+
         namespace, name = self._getNsTag(name)
-        if namespace is not None:
-            for inverted_nsmap in reversed(self.nsmaps):
-                if inverted_nsmap is not None and namespace in inverted_nsmap:
-                    nsprefix = inverted_nsmap[namespace]
-                    break
+        nsprefix = self._prefix_for_namespace(namespace)
         self.soup.handle_starttag(name, namespace, nsprefix, attrs)
 
+    def _prefix_for_namespace(self, namespace):
+        """Find the currently active prefix for the given namespace."""
+        if namespace is None:
+            return None
+        for inverted_nsmap in reversed(self.nsmaps):
+            if inverted_nsmap is not None and namespace in inverted_nsmap:
+                return inverted_nsmap[namespace]
+        return None
+
     def end(self, name):
         self.soup.endData()
         completed_tag = self.soup.tagStack[-1]
@@ -130,14 +183,10 @@ class LXMLTreeBuilderForXML(TreeBuilder):
                     nsprefix = inverted_nsmap[namespace]
                     break
         self.soup.handle_endtag(name, nsprefix)
-        if self.nsmaps != None:
+        if len(self.nsmaps) > 1:
             # This tag, or one of its parents, introduced a namespace
             # mapping, so pop it off the stack.
             self.nsmaps.pop()
-            if len(self.nsmaps) == 0:
-                # Namespaces are no longer in play, so don't bother keeping
-                # track of the namespace stack.
-                self.nsmaps = None
 
     def pi(self, target, data):
         pass
@@ -166,13 +215,18 @@ class LXMLTreeBuilder(HTMLTreeBuilder, LXMLTreeBuilderForXML):
     features = [LXML, HTML, FAST, PERMISSIVE]
     is_xml = False
 
-    @property
-    def default_parser(self):
+    def default_parser(self, encoding):
         return etree.HTMLParser
 
     def feed(self, markup):
-        self.parser.feed(markup)
-        self.parser.close()
+        encoding = self.soup.original_encoding
+        try:
+            self.parser = self.parser_for(encoding)
+            self.parser.feed(markup)
+            self.parser.close()
+        except (UnicodeDecodeError, LookupError, etree.ParserError), e:
+            raise ParserRejectedMarkup(str(e))
+
 
     def test_fragment_to_document(self, fragment):
         """See `TreeBuilder`."""

libs/bs4/dammit.py

@@ -1,27 +1,40 @@
 # -*- coding: utf-8 -*-
 """Beautiful Soup bonus library: Unicode, Dammit
 
-This class forces XML data into a standard format (usually to UTF-8 or
-Unicode).  It is heavily based on code from Mark Pilgrim's Universal
-Feed Parser. It does not rewrite the XML or HTML to reflect a new
-encoding; that's the tree builder's job.
+This library converts a bytestream to Unicode through any means
+necessary. It is heavily based on code from Mark Pilgrim's Universal
+Feed Parser. It works best on XML and XML, but it does not rewrite the
+XML or HTML to reflect a new encoding; that's the tree builder's job.
 """
 
 import codecs
 from htmlentitydefs import codepoint2name
 import re
-import warnings
+import logging
+import string
 
-# Autodetects character encodings. Very useful.
-# Download from http://chardet.feedparser.org/
-#  or 'apt-get install python-chardet'
-#  or 'easy_install chardet'
+# Import a library to autodetect character encodings.
+chardet_type = None
 try:
-    import chardet
-    #import chardet.constants
-    #chardet.constants._debug = 1
+    # First try the fast C implementation.
+    #  PyPI package: cchardet
+    import cchardet
+    def chardet_dammit(s):
+        return cchardet.detect(s)['encoding']
 except ImportError:
-    chardet = None
+    try:
+        # Fall back to the pure Python implementation
+        #  Debian package: python-chardet
+        #  PyPI package: chardet
+        import chardet
+        def chardet_dammit(s):
+            return chardet.detect(s)['encoding']
+        #import chardet.constants
+        #chardet.constants._debug = 1
+    except ImportError:
+        # No chardet available.
+        def chardet_dammit(s):
+            return None
 
 # Available from http://cjkpython.i18n.org/.
 try:
@@ -69,6 +82,8 @@ class EntitySubstitution(object):
                                            "&(?!#\d+;|#x[0-9a-fA-F]+;|\w+;)"
                                            ")")
 
+    AMPERSAND_OR_BRACKET = re.compile("([<>&])")
+
     @classmethod
     def _substitute_html_entity(cls, matchobj):
         entity = cls.CHARACTER_TO_HTML_ENTITY.get(matchobj.group(0))
@@ -122,6 +137,28 @@ class EntitySubstitution(object):
     def substitute_xml(cls, value, make_quoted_attribute=False):
         """Substitute XML entities for special XML characters.
 
+        :param value: A string to be substituted. The less-than sign
+          will become &lt;, the greater-than sign will become &gt;,
+          and any ampersands will become &amp;. If you want ampersands
+          that appear to be part of an entity definition to be left
+          alone, use substitute_xml_containing_entities() instead.
+
+        :param make_quoted_attribute: If True, then the string will be
+         quoted, as befits an attribute value.
+        """
+        # Escape angle brackets and ampersands.
+        value = cls.AMPERSAND_OR_BRACKET.sub(
+            cls._substitute_xml_entity, value)
+
+        if make_quoted_attribute:
+            value = cls.quoted_attribute_value(value)
+        return value
+
+    @classmethod
+    def substitute_xml_containing_entities(
+        cls, value, make_quoted_attribute=False):
+        """Substitute XML entities for special XML characters.
+
         :param value: A string to be substituted. The less-than sign will
           become &lt;, the greater-than sign will become &gt;, and any
           ampersands that are not part of an entity defition will
@@ -155,6 +192,125 @@ class EntitySubstitution(object):
             cls._substitute_html_entity, s)
 
 
+class EncodingDetector:
+    """Suggests a number of possible encodings for a bytestring.
+
+    Order of precedence:
+
+    1. Encodings you specifically tell EncodingDetector to try first
+    (the override_encodings argument to the constructor).
+
+    2. An encoding declared within the bytestring itself, either in an
+    XML declaration (if the bytestring is to be interpreted as an XML
+    document), or in a <meta> tag (if the bytestring is to be
+    interpreted as an HTML document.)
+
+    3. An encoding detected through textual analysis by chardet,
+    cchardet, or a similar external library.
+
+    4. UTF-8.
+
+    5. Windows-1252.
+    """
+    def __init__(self, markup, override_encodings=None, is_html=False):
+        self.override_encodings = override_encodings or []
+        self.chardet_encoding = None
+        self.is_html = is_html
+        self.declared_encoding = None
+
+        # First order of business: strip a byte-order mark.
+        self.markup, self.sniffed_encoding = self.strip_byte_order_mark(markup)
+
+    def _usable(self, encoding, tried):
+        if encoding is not None:
+            encoding = encoding.lower()
+            if encoding not in tried:
+                tried.add(encoding)
+                return True
+        return False
+
+    @property
+    def encodings(self):
+        """Yield a number of encodings that might work for this markup."""
+        tried = set()
+        for e in self.override_encodings:
+            if self._usable(e, tried):
+                yield e
+
+        # Did the document originally start with a byte-order mark
+        # that indicated its encoding?
+        if self._usable(self.sniffed_encoding, tried):
+            yield self.sniffed_encoding
+
+        # Look within the document for an XML or HTML encoding
+        # declaration.
+        if self.declared_encoding is None:
+            self.declared_encoding = self.find_declared_encoding(
+                self.markup, self.is_html)
+        if self._usable(self.declared_encoding, tried):
+            yield self.declared_encoding
+
+        # Use third-party character set detection to guess at the
+        # encoding.
+        if self.chardet_encoding is None:
+            self.chardet_encoding = chardet_dammit(self.markup)
+        if self._usable(self.chardet_encoding, tried):
+            yield self.chardet_encoding
+
+        # As a last-ditch effort, try utf-8 and windows-1252.
+        for e in ('utf-8', 'windows-1252'):
+            if self._usable(e, tried):
+                yield e
+
+    @classmethod
+    def strip_byte_order_mark(cls, data):
+        """If a byte-order mark is present, strip it and return the encoding it implies."""
+        encoding = None
+        if (len(data) >= 4) and (data[:2] == b'\xfe\xff') \
+               and (data[2:4] != '\x00\x00'):
+            encoding = 'utf-16be'
+            data = data[2:]
+        elif (len(data) >= 4) and (data[:2] == b'\xff\xfe') \
+                 and (data[2:4] != '\x00\x00'):
+            encoding = 'utf-16le'
+            data = data[2:]
+        elif data[:3] == b'\xef\xbb\xbf':
+            encoding = 'utf-8'
+            data = data[3:]
+        elif data[:4] == b'\x00\x00\xfe\xff':
+            encoding = 'utf-32be'
+            data = data[4:]
+        elif data[:4] == b'\xff\xfe\x00\x00':
+            encoding = 'utf-32le'
+            data = data[4:]
+        return data, encoding
+
+    @classmethod
+    def find_declared_encoding(cls, markup, is_html=False, search_entire_document=False):
+        """Given a document, tries to find its declared encoding.
+
+        An XML encoding is declared at the beginning of the document.
+
+        An HTML encoding is declared in a <meta> tag, hopefully near the
+        beginning of the document.
+        """
+        if search_entire_document:
+            xml_endpos = html_endpos = len(markup)
+        else:
+            xml_endpos = 1024
+            html_endpos = max(2048, int(len(markup) * 0.05))
+            
+        declared_encoding = None
+        declared_encoding_match = xml_encoding_re.search(markup, endpos=xml_endpos)
+        if not declared_encoding_match and is_html:
+            declared_encoding_match = html_meta_re.search(markup, endpos=html_endpos)
+        if declared_encoding_match is not None:
+            declared_encoding = declared_encoding_match.groups()[0].decode(
+                'ascii')
+        if declared_encoding:
+            return declared_encoding.lower()
+        return None
+
 class UnicodeDammit:
     """A class for detecting the encoding of a *ML document and
     converting it to a Unicode string. If the source encoding is
@@ -176,65 +332,48 @@ class UnicodeDammit:
 
     def __init__(self, markup, override_encodings=[],
                  smart_quotes_to=None, is_html=False):
-        self.declared_html_encoding = None
         self.smart_quotes_to = smart_quotes_to
         self.tried_encodings = []
         self.contains_replacement_characters = False
+        self.is_html = is_html
 
-        if markup == '' or isinstance(markup, unicode):
+        self.detector = EncodingDetector(markup, override_encodings, is_html)
+
+        # Short-circuit if the data is in Unicode to begin with.
+        if isinstance(markup, unicode) or markup == '':
             self.markup = markup
             self.unicode_markup = unicode(markup)
             self.original_encoding = None
             return
 
-        new_markup, document_encoding, sniffed_encoding = \
-            self._detectEncoding(markup, is_html)
-        self.markup = new_markup
+        # The encoding detector may have stripped a byte-order mark.
+        # Use the stripped markup from this point on.
+        self.markup = self.detector.markup
 
         u = None
-        if new_markup != markup:
-            # _detectEncoding modified the markup, then converted it to
-            # Unicode and then to UTF-8. So convert it from UTF-8.
-            u = self._convert_from("utf8")
-            self.original_encoding = sniffed_encoding
+        for encoding in self.detector.encodings:
+            markup = self.detector.markup
+            u = self._convert_from(encoding)
+            if u is not None:
+                break
 
         if not u:
-            for proposed_encoding in (
-                override_encodings + [document_encoding, sniffed_encoding]):
-                if proposed_encoding is not None:
-                    u = self._convert_from(proposed_encoding)
-                    if u:
-                        break
+            # None of the encodings worked. As an absolute last resort,
+            # try them again with character replacement.
 
-        # If no luck and we have auto-detection library, try that:
-        if not u and chardet and not isinstance(self.markup, unicode):
-            u = self._convert_from(chardet.detect(self.markup)['encoding'])
-
-        # As a last resort, try utf-8 and windows-1252:
-        if not u:
-            for proposed_encoding in ("utf-8", "windows-1252"):
-                u = self._convert_from(proposed_encoding)
-                if u:
-                    break
-
-        # As an absolute last resort, try the encodings again with
-        # character replacement.
-        if not u:
-            for proposed_encoding in (
-                override_encodings + [
-                    document_encoding, sniffed_encoding, "utf-8", "windows-1252"]):
-                if proposed_encoding != "ascii":
-                    u = self._convert_from(proposed_encoding, "replace")
+            for encoding in self.detector.encodings:
+                if encoding != "ascii":
+                    u = self._convert_from(encoding, "replace")
                 if u is not None:
-                    warnings.warn(
-                        UnicodeWarning(
+                    logging.warning(
                             "Some characters could not be decoded, and were "
-                            "replaced with REPLACEMENT CHARACTER."))
+                            "replaced with REPLACEMENT CHARACTER.")
                     self.contains_replacement_characters = True
                     break
 
-        # We could at this point force it to ASCII, but that would
-        # destroy so much data that I think giving up is better
+        # If none of that worked, we could at this point force it to
+        # ASCII, but that would destroy so much data that I think
+        # giving up is better.
         self.unicode_markup = u
         if not u:
             self.original_encoding = None
@@ -262,11 +401,10 @@ class UnicodeDammit:
             return None
         self.tried_encodings.append((proposed, errors))
         markup = self.markup
-
         # Convert smart quotes to HTML if coming from an encoding
         # that might have them.
         if (self.smart_quotes_to is not None
-            and proposed.lower() in self.ENCODINGS_WITH_SMART_QUOTES):
+            and proposed in self.ENCODINGS_WITH_SMART_QUOTES):
             smart_quotes_re = b"([\x80-\x9f])"
             smart_quotes_compiled = re.compile(smart_quotes_re)
             markup = smart_quotes_compiled.sub(self._sub_ms_char, markup)
@@ -287,99 +425,24 @@ class UnicodeDammit:
     def _to_unicode(self, data, encoding, errors="strict"):
         '''Given a string and its encoding, decodes the string into Unicode.
         %encoding is a string recognized by encodings.aliases'''
+        return unicode(data, encoding, errors)
 
-        # strip Byte Order Mark (if present)
-        if (len(data) >= 4) and (data[:2] == '\xfe\xff') \
-               and (data[2:4] != '\x00\x00'):
-            encoding = 'utf-16be'
-            data = data[2:]
-        elif (len(data) >= 4) and (data[:2] == '\xff\xfe') \
-                 and (data[2:4] != '\x00\x00'):
-            encoding = 'utf-16le'
-            data = data[2:]
-        elif data[:3] == '\xef\xbb\xbf':
-            encoding = 'utf-8'
-            data = data[3:]
-        elif data[:4] == '\x00\x00\xfe\xff':
-            encoding = 'utf-32be'
-            data = data[4:]
-        elif data[:4] == '\xff\xfe\x00\x00':
-            encoding = 'utf-32le'
-            data = data[4:]
-        newdata = unicode(data, encoding, errors)
-        return newdata
-
-    def _detectEncoding(self, xml_data, is_html=False):
-        """Given a document, tries to detect its XML encoding."""
-        xml_encoding = sniffed_xml_encoding = None
-        try:
-            if xml_data[:4] == b'\x4c\x6f\xa7\x94':
-                # EBCDIC
-                xml_data = self._ebcdic_to_ascii(xml_data)
-            elif xml_data[:4] == b'\x00\x3c\x00\x3f':
-                # UTF-16BE
-                sniffed_xml_encoding = 'utf-16be'
-                xml_data = unicode(xml_data, 'utf-16be').encode('utf-8')
-            elif (len(xml_data) >= 4) and (xml_data[:2] == b'\xfe\xff') \
-                     and (xml_data[2:4] != b'\x00\x00'):
-                # UTF-16BE with BOM
-                sniffed_xml_encoding = 'utf-16be'
-                xml_data = unicode(xml_data[2:], 'utf-16be').encode('utf-8')
-            elif xml_data[:4] == b'\x3c\x00\x3f\x00':
-                # UTF-16LE
-                sniffed_xml_encoding = 'utf-16le'
-                xml_data = unicode(xml_data, 'utf-16le').encode('utf-8')
-            elif (len(xml_data) >= 4) and (xml_data[:2] == b'\xff\xfe') and \
-                     (xml_data[2:4] != b'\x00\x00'):
-                # UTF-16LE with BOM
-                sniffed_xml_encoding = 'utf-16le'
-                xml_data = unicode(xml_data[2:], 'utf-16le').encode('utf-8')
-            elif xml_data[:4] == b'\x00\x00\x00\x3c':
-                # UTF-32BE
-                sniffed_xml_encoding = 'utf-32be'
-                xml_data = unicode(xml_data, 'utf-32be').encode('utf-8')
-            elif xml_data[:4] == b'\x3c\x00\x00\x00':
-                # UTF-32LE
-                sniffed_xml_encoding = 'utf-32le'
-                xml_data = unicode(xml_data, 'utf-32le').encode('utf-8')
-            elif xml_data[:4] == b'\x00\x00\xfe\xff':
-                # UTF-32BE with BOM
-                sniffed_xml_encoding = 'utf-32be'
-                xml_data = unicode(xml_data[4:], 'utf-32be').encode('utf-8')
-            elif xml_data[:4] == b'\xff\xfe\x00\x00':
-                # UTF-32LE with BOM
-                sniffed_xml_encoding = 'utf-32le'
-                xml_data = unicode(xml_data[4:], 'utf-32le').encode('utf-8')
-            elif xml_data[:3] == b'\xef\xbb\xbf':
-                # UTF-8 with BOM
-                sniffed_xml_encoding = 'utf-8'
-                xml_data = unicode(xml_data[3:], 'utf-8').encode('utf-8')
-            else:
-                sniffed_xml_encoding = 'ascii'
-                pass
-        except:
-            xml_encoding_match = None
-        xml_encoding_match = xml_encoding_re.match(xml_data)
-        if not xml_encoding_match and is_html:
-            xml_encoding_match = html_meta_re.search(xml_data)
-        if xml_encoding_match is not None:
-            xml_encoding = xml_encoding_match.groups()[0].decode(
-                'ascii').lower()
-            if is_html:
-                self.declared_html_encoding = xml_encoding
-            if sniffed_xml_encoding and \
-               (xml_encoding in ('iso-10646-ucs-2', 'ucs-2', 'csunicode',
-                                 'iso-10646-ucs-4', 'ucs-4', 'csucs4',
-                                 'utf-16', 'utf-32', 'utf_16', 'utf_32',
-                                 'utf16', 'u16')):
-                xml_encoding = sniffed_xml_encoding
-        return xml_data, xml_encoding, sniffed_xml_encoding
+    @property
+    def declared_html_encoding(self):
+        if not self.is_html:
+            return None
+        return self.detector.declared_encoding
 
     def find_codec(self, charset):
-        return self._codec(self.CHARSET_ALIASES.get(charset, charset)) \
-               or (charset and self._codec(charset.replace("-", ""))) \
-               or (charset and self._codec(charset.replace("-", "_"))) \
+        value = (self._codec(self.CHARSET_ALIASES.get(charset, charset))
+               or (charset and self._codec(charset.replace("-", "")))
+               or (charset and self._codec(charset.replace("-", "_")))
+               or (charset and charset.lower())
                or charset
+                )
+        if value:
+            return value.lower()
+        return None
 
     def _codec(self, charset):
         if not charset:
@@ -392,32 +455,6 @@ class UnicodeDammit:
             pass
         return codec
 
-    EBCDIC_TO_ASCII_MAP = None
-
-    def _ebcdic_to_ascii(self, s):
-        c = self.__class__
-        if not c.EBCDIC_TO_ASCII_MAP:
-            emap = (0,1,2,3,156,9,134,127,151,141,142,11,12,13,14,15,
-                    16,17,18,19,157,133,8,135,24,25,146,143,28,29,30,31,
-                    128,129,130,131,132,10,23,27,136,137,138,139,140,5,6,7,
-                    144,145,22,147,148,149,150,4,152,153,154,155,20,21,158,26,
-                    32,160,161,162,163,164,165,166,167,168,91,46,60,40,43,33,
-                    38,169,170,171,172,173,174,175,176,177,93,36,42,41,59,94,
-                    45,47,178,179,180,181,182,183,184,185,124,44,37,95,62,63,
-                    186,187,188,189,190,191,192,193,194,96,58,35,64,39,61,34,
-                    195,97,98,99,100,101,102,103,104,105,196,197,198,199,200,
-                    201,202,106,107,108,109,110,111,112,113,114,203,204,205,
-                    206,207,208,209,126,115,116,117,118,119,120,121,122,210,
-                    211,212,213,214,215,216,217,218,219,220,221,222,223,224,
-                    225,226,227,228,229,230,231,123,65,66,67,68,69,70,71,72,
-                    73,232,233,234,235,236,237,125,74,75,76,77,78,79,80,81,
-                    82,238,239,240,241,242,243,92,159,83,84,85,86,87,88,89,
-                    90,244,245,246,247,248,249,48,49,50,51,52,53,54,55,56,57,
-                    250,251,252,253,254,255)
-            import string
-            c.EBCDIC_TO_ASCII_MAP = string.maketrans(
-            ''.join(map(chr, list(range(256)))), ''.join(map(chr, emap)))
-        return s.translate(c.EBCDIC_TO_ASCII_MAP)
 
     # A partial mapping of ISO-Latin-1 to HTML entities/XML numeric entities.
     MS_CHARS = {b'\x80': ('euro', '20AC'),

libs/bs4/diagnose.py

@@ -0,0 +1,204 @@
+"""Diagnostic functions, mainly for use when doing tech support."""
+import cProfile
+from StringIO import StringIO
+from HTMLParser import HTMLParser
+import bs4
+from bs4 import BeautifulSoup, __version__
+from bs4.builder import builder_registry
+
+import os
+import pstats
+import random
+import tempfile
+import time
+import traceback
+import sys
+import cProfile
+
+def diagnose(data):
+    """Diagnostic suite for isolating common problems."""
+    print "Diagnostic running on Beautiful Soup %s" % __version__
+    print "Python version %s" % sys.version
+
+    basic_parsers = ["html.parser", "html5lib", "lxml"]
+    for name in basic_parsers:
+        for builder in builder_registry.builders:
+            if name in builder.features:
+                break
+        else:
+            basic_parsers.remove(name)
+            print (
+                "I noticed that %s is not installed. Installing it may help." %
+                name)
+
+    if 'lxml' in basic_parsers:
+        basic_parsers.append(["lxml", "xml"])
+        from lxml import etree
+        print "Found lxml version %s" % ".".join(map(str,etree.LXML_VERSION))
+
+    if 'html5lib' in basic_parsers:
+        import html5lib
+        print "Found html5lib version %s" % html5lib.__version__
+
+    if hasattr(data, 'read'):
+        data = data.read()
+    elif os.path.exists(data):
+        print '"%s" looks like a filename. Reading data from the file.' % data
+        data = open(data).read()
+    elif data.startswith("http:") or data.startswith("https:"):
+        print '"%s" looks like a URL. Beautiful Soup is not an HTTP client.' % data
+        print "You need to use some other library to get the document behind the URL, and feed that document to Beautiful Soup."
+        return
+    print
+
+    for parser in basic_parsers:
+        print "Trying to parse your markup with %s" % parser
+        success = False
+        try:
+            soup = BeautifulSoup(data, parser)
+            success = True
+        except Exception, e:
+            print "%s could not parse the markup." % parser
+            traceback.print_exc()
+        if success:
+            print "Here's what %s did with the markup:" % parser
+            print soup.prettify()
+
+        print "-" * 80
+
+def lxml_trace(data, html=True, **kwargs):
+    """Print out the lxml events that occur during parsing.
+
+    This lets you see how lxml parses a document when no Beautiful
+    Soup code is running.
+    """
+    from lxml import etree
+    for event, element in etree.iterparse(StringIO(data), html=html, **kwargs):
+        print("%s, %4s, %s" % (event, element.tag, element.text))
+
+class AnnouncingParser(HTMLParser):
+    """Announces HTMLParser parse events, without doing anything else."""
+
+    def _p(self, s):
+        print(s)
+
+    def handle_starttag(self, name, attrs):
+        self._p("%s START" % name)
+
+    def handle_endtag(self, name):
+        self._p("%s END" % name)
+
+    def handle_data(self, data):
+        self._p("%s DATA" % data)
+
+    def handle_charref(self, name):
+        self._p("%s CHARREF" % name)
+
+    def handle_entityref(self, name):
+        self._p("%s ENTITYREF" % name)
+
+    def handle_comment(self, data):
+        self._p("%s COMMENT" % data)
+
+    def handle_decl(self, data):
+        self._p("%s DECL" % data)
+
+    def unknown_decl(self, data):
+        self._p("%s UNKNOWN-DECL" % data)
+
+    def handle_pi(self, data):
+        self._p("%s PI" % data)
+
+def htmlparser_trace(data):
+    """Print out the HTMLParser events that occur during parsing.
+
+    This lets you see how HTMLParser parses a document when no
+    Beautiful Soup code is running.
+    """
+    parser = AnnouncingParser()
+    parser.feed(data)
+
+_vowels = "aeiou"
+_consonants = "bcdfghjklmnpqrstvwxyz"
+
+def rword(length=5):
+    "Generate a random word-like string."
+    s = ''
+    for i in range(length):
+        if i % 2 == 0:
+            t = _consonants
+        else:
+            t = _vowels
+        s += random.choice(t)
+    return s
+
+def rsentence(length=4):
+    "Generate a random sentence-like string."
+    return " ".join(rword(random.randint(4,9)) for i in range(length))
+        
+def rdoc(num_elements=1000):
+    """Randomly generate an invalid HTML document."""
+    tag_names = ['p', 'div', 'span', 'i', 'b', 'script', 'table']
+    elements = []
+    for i in range(num_elements):
+        choice = random.randint(0,3)
+        if choice == 0:
+            # New tag.
+            tag_name = random.choice(tag_names)
+            elements.append("<%s>" % tag_name)
+        elif choice == 1:
+            elements.append(rsentence(random.randint(1,4)))
+        elif choice == 2:
+            # Close a tag.
+            tag_name = random.choice(tag_names)
+            elements.append("</%s>" % tag_name)
+    return "<html>" + "\n".join(elements) + "</html>"
+
+def benchmark_parsers(num_elements=100000):
+    """Very basic head-to-head performance benchmark."""
+    print "Comparative parser benchmark on Beautiful Soup %s" % __version__
+    data = rdoc(num_elements)
+    print "Generated a large invalid HTML document (%d bytes)." % len(data)
+    
+    for parser in ["lxml", ["lxml", "html"], "html5lib", "html.parser"]:
+        success = False
+        try:
+            a = time.time()
+            soup = BeautifulSoup(data, parser)
+            b = time.time()
+            success = True
+        except Exception, e:
+            print "%s could not parse the markup." % parser
+            traceback.print_exc()
+        if success:
+            print "BS4+%s parsed the markup in %.2fs." % (parser, b-a)
+
+    from lxml import etree
+    a = time.time()
+    etree.HTML(data)
+    b = time.time()
+    print "Raw lxml parsed the markup in %.2fs." % (b-a)
+
+    import html5lib
+    parser = html5lib.HTMLParser()
+    a = time.time()
+    parser.parse(data)
+    b = time.time()
+    print "Raw html5lib parsed the markup in %.2fs." % (b-a)
+
+def profile(num_elements=100000, parser="lxml"):
+
+    filehandle = tempfile.NamedTemporaryFile()
+    filename = filehandle.name
+
+    data = rdoc(num_elements)
+    vars = dict(bs4=bs4, data=data, parser=parser)
+    cProfile.runctx('bs4.BeautifulSoup(data, parser)' , vars, vars, filename)
+
+    stats = pstats.Stats(filename)
+    # stats.strip_dirs()
+    stats.sort_stats("cumulative")
+    stats.print_stats('_html5lib|bs4', 50)
+
+if __name__ == '__main__':
+    diagnose(sys.stdin.read())

libs/bs4/element.py

@@ -26,6 +26,9 @@ class NamespacedAttribute(unicode):
     def __new__(cls, prefix, name, namespace=None):
         if name is None:
             obj = unicode.__new__(cls, prefix)
+        elif prefix is None:
+            # Not really namespaced.
+            obj = unicode.__new__(cls, name)
         else:
             obj = unicode.__new__(cls, prefix + ":" + name)
         obj.prefix = prefix
@@ -78,6 +81,40 @@ class ContentMetaAttributeValue(AttributeValueWithCharsetSubstitution):
             return match.group(1) + encoding
         return self.CHARSET_RE.sub(rewrite, self.original_value)
 
+class HTMLAwareEntitySubstitution(EntitySubstitution):
+
+    """Entity substitution rules that are aware of some HTML quirks.
+
+    Specifically, the contents of <script> and <style> tags should not
+    undergo entity substitution.
+
+    Incoming NavigableString objects are checked to see if they're the
+    direct children of a <script> or <style> tag.
+    """
+
+    cdata_containing_tags = set(["script", "style"])
+
+    preformatted_tags = set(["pre"])
+
+    @classmethod
+    def _substitute_if_appropriate(cls, ns, f):
+        if (isinstance(ns, NavigableString)
+            and ns.parent is not None
+            and ns.parent.name in cls.cdata_containing_tags):
+            # Do nothing.
+            return ns
+        # Substitute.
+        return f(ns)
+
+    @classmethod
+    def substitute_html(cls, ns):
+        return cls._substitute_if_appropriate(
+            ns, EntitySubstitution.substitute_html)
+
+    @classmethod
+    def substitute_xml(cls, ns):
+        return cls._substitute_if_appropriate(
+            ns, EntitySubstitution.substitute_xml)
 
 class PageElement(object):
     """Contains the navigational information for some part of the page
@@ -94,25 +131,60 @@ class PageElement(object):
     #   converted to entities.  This is not recommended, but it's
     #   faster than "minimal".
     # A function - This function will be called on every string that
-    #  needs to undergo entity substition
-    FORMATTERS = {
+    #  needs to undergo entity substitution.
+    #
+
+    # In an HTML document, the default "html" and "minimal" functions
+    # will leave the contents of <script> and <style> tags alone. For
+    # an XML document, all tags will be given the same treatment.
+
+    HTML_FORMATTERS = {
+        "html" : HTMLAwareEntitySubstitution.substitute_html,
+        "minimal" : HTMLAwareEntitySubstitution.substitute_xml,
+        None : None
+        }
+
+    XML_FORMATTERS = {
         "html" : EntitySubstitution.substitute_html,
         "minimal" : EntitySubstitution.substitute_xml,
         None : None
         }
 
-    @classmethod
     def format_string(self, s, formatter='minimal'):
         """Format the given string using the given formatter."""
         if not callable(formatter):
-            formatter = self.FORMATTERS.get(
-                formatter, EntitySubstitution.substitute_xml)
+            formatter = self._formatter_for_name(formatter)
         if formatter is None:
             output = s
         else:
             output = formatter(s)
         return output
 
+    @property
+    def _is_xml(self):
+        """Is this element part of an XML tree or an HTML tree?
+
+        This is used when mapping a formatter name ("minimal") to an
+        appropriate function (one that performs entity-substitution on
+        the contents of <script> and <style> tags, or not). It's
+        inefficient, but it should be called very rarely.
+        """
+        if self.parent is None:
+            # This is the top-level object. It should have .is_xml set
+            # from tree creation. If not, take a guess--BS is usually
+            # used on HTML markup.
+            return getattr(self, 'is_xml', False)
+        return self.parent._is_xml
+
+    def _formatter_for_name(self, name):
+        "Look up a formatter function based on its name and the tree."
+        if self._is_xml:
+            return self.XML_FORMATTERS.get(
+                name, EntitySubstitution.substitute_xml)
+        else:
+            return self.HTML_FORMATTERS.get(
+                name, HTMLAwareEntitySubstitution.substitute_xml)
+
     def setup(self, parent=None, previous_element=None):
         """Sets up the initial relations between this element and
         other elements."""
@@ -183,11 +255,16 @@ class PageElement(object):
         self.previous_sibling = self.next_sibling = None
         return self
 
-    def _last_descendant(self):
+    def _last_descendant(self, is_initialized=True, accept_self=True):
         "Finds the last element beneath this object to be parsed."
-        last_child = self
-        while hasattr(last_child, 'contents') and last_child.contents:
-            last_child = last_child.contents[-1]
+        if is_initialized and self.next_sibling:
+            last_child = self.next_sibling.previous_element
+        else:
+            last_child = self
+            while isinstance(last_child, Tag) and last_child.contents:
+                last_child = last_child.contents[-1]
+        if not accept_self and last_child == self:
+            last_child = None
         return last_child
     # BS3: Not part of the API!
     _lastRecursiveChild = _last_descendant
@@ -222,11 +299,11 @@ class PageElement(object):
             previous_child = self.contents[position - 1]
             new_child.previous_sibling = previous_child
             new_child.previous_sibling.next_sibling = new_child
-            new_child.previous_element = previous_child._last_descendant()
+            new_child.previous_element = previous_child._last_descendant(False)
         if new_child.previous_element is not None:
             new_child.previous_element.next_element = new_child
 
-        new_childs_last_element = new_child._last_descendant()
+        new_childs_last_element = new_child._last_descendant(False)
 
         if position >= len(self.contents):
             new_child.next_sibling = None
@@ -366,7 +443,7 @@ class PageElement(object):
         # NOTE: We can't use _find_one because findParents takes a different
         # set of arguments.
         r = None
-        l = self.find_parents(name, attrs, 1)
+        l = self.find_parents(name, attrs, 1, **kwargs)
         if l:
             r = l[0]
         return r
@@ -403,20 +480,21 @@ class PageElement(object):
 
         if isinstance(name, SoupStrainer):
             strainer = name
-        elif text is None and not limit and not attrs and not kwargs:
-            # Optimization to find all tags.
-            if name is True or name is None:
-                return [element for element in generator
-                        if isinstance(element, Tag)]
-            # Optimization to find all tags with a given name.
-            elif isinstance(name, basestring):
-                return [element for element in generator
-                        if isinstance(element, Tag) and element.name == name]
-            else:
-                strainer = SoupStrainer(name, attrs, text, **kwargs)
         else:
-            # Build a SoupStrainer
             strainer = SoupStrainer(name, attrs, text, **kwargs)
+
+        if text is None and not limit and not attrs and not kwargs:
+            if name is True or name is None:
+                # Optimization to find all tags.
+                result = (element for element in generator
+                          if isinstance(element, Tag))
+                return ResultSet(strainer, result)
+            elif isinstance(name, basestring):
+                # Optimization to find all tags with a given name.
+                result = (element for element in generator
+                          if isinstance(element, Tag)
+                            and element.name == name)
+                return ResultSet(strainer, result)
         results = ResultSet(strainer)
         while True:
             try:
@@ -495,6 +573,14 @@ class PageElement(object):
             value =" ".join(value)
         return value
 
+    def _tag_name_matches_and(self, function, tag_name):
+        if not tag_name:
+            return function
+        else:
+            def _match(tag):
+                return tag.name == tag_name and function(tag)
+            return _match
+
     def _attribute_checker(self, operator, attribute, value=''):
         """Create a function that performs a CSS selector operation.
 
@@ -536,87 +622,6 @@ class PageElement(object):
         else:
             return lambda el: el.has_attr(attribute)
 
-    def select(self, selector):
-        """Perform a CSS selection operation on the current element."""
-        tokens = selector.split()
-        current_context = [self]
-        for index, token in enumerate(tokens):
-            if tokens[index - 1] == '>':
-                # already found direct descendants in last step. skip this
-                # step.
-                continue
-            m = self.attribselect_re.match(token)
-            if m is not None:
-                # Attribute selector
-                tag, attribute, operator, value = m.groups()
-                if not tag:
-                    tag = True
-                checker = self._attribute_checker(operator, attribute, value)
-                found = []
-                for context in current_context:
-                    found.extend(
-                        [el for el in context.find_all(tag) if checker(el)])
-                current_context = found
-                continue
-
-            if '#' in token:
-                # ID selector
-                tag, id = token.split('#', 1)
-                if tag == "":
-                    tag = True
-                el = current_context[0].find(tag, {'id': id})
-                if el is None:
-                    return [] # No match
-                current_context = [el]
-                continue
-
-            if '.' in token:
-                # Class selector
-                tag_name, klass = token.split('.', 1)
-                if not tag_name:
-                    tag_name = True
-                classes = set(klass.split('.'))
-                found = []
-                def classes_match(tag):
-                    if tag_name is not True and tag.name != tag_name:
-                        return False
-                    if not tag.has_attr('class'):
-                        return False
-                    return classes.issubset(tag['class'])
-                for context in current_context:
-                    found.extend(context.find_all(classes_match))
-                current_context = found
-                continue
-
-            if token == '*':
-                # Star selector
-                found = []
-                for context in current_context:
-                    found.extend(context.findAll(True))
-                current_context = found
-                continue
-
-            if token == '>':
-                # Child selector
-                tag = tokens[index + 1]
-                if not tag:
-                    tag = True
-
-                found = []
-                for context in current_context:
-                    found.extend(context.find_all(tag, recursive=False))
-                current_context = found
-                continue
-
-            # Here we should just have a regular tag
-            if not self.tag_name_re.match(token):
-                return []
-            found = []
-            for context in current_context:
-                found.extend(context.findAll(token))
-            current_context = found
-        return current_context
-
     # Old non-property versions of the generators, for backwards
     # compatibility with BS3.
     def nextGenerator(self):
@@ -652,6 +657,9 @@ class NavigableString(unicode, PageElement):
             return unicode.__new__(cls, value)
         return unicode.__new__(cls, value, DEFAULT_OUTPUT_ENCODING)
 
+    def __copy__(self):
+        return self
+
     def __getnewargs__(self):
         return (unicode(self),)
 
@@ -670,6 +678,13 @@ class NavigableString(unicode, PageElement):
         output = self.format_string(self, formatter)
         return self.PREFIX + output + self.SUFFIX
 
+    @property
+    def name(self):
+        return None
+
+    @name.setter
+    def name(self, name):
+        raise AttributeError("A NavigableString cannot be given a name.")
 
 class PreformattedString(NavigableString):
     """A NavigableString not subject to the normal formatting rules.
@@ -709,7 +724,7 @@ class Doctype(PreformattedString):
 
     @classmethod
     def for_name_and_ids(cls, name, pub_id, system_id):
-        value = name
+        value = name or ''
         if pub_id is not None:
             value += ' PUBLIC "%s"' % pub_id
             if system_id is not None:
@@ -744,7 +759,7 @@ class Tag(PageElement):
         self.prefix = prefix
         if attrs is None:
             attrs = {}
-        elif builder.cdata_list_attributes:
+        elif attrs and builder.cdata_list_attributes:
             attrs = builder._replace_cdata_list_attribute_values(
                 self.name, attrs)
         else:
@@ -803,16 +818,24 @@ class Tag(PageElement):
         self.clear()
         self.append(string.__class__(string))
 
-    def _all_strings(self, strip=False):
-        """Yield all child strings, possibly stripping them."""
+    def _all_strings(self, strip=False, types=(NavigableString, CData)):
+        """Yield all strings of certain classes, possibly stripping them.
+
+        By default, yields only NavigableString and CData objects. So
+        no comments, processing instructions, etc.
+        """
         for descendant in self.descendants:
-            if not isinstance(descendant, NavigableString):
+            if (
+                (types is None and not isinstance(descendant, NavigableString))
+                or
+                (types is not None and type(descendant) not in types)):
                 continue
             if strip:
                 descendant = descendant.strip()
                 if len(descendant) == 0:
                     continue
             yield descendant
+
     strings = property(_all_strings)
 
     @property
@@ -820,11 +843,13 @@ class Tag(PageElement):
         for string in self._all_strings(True):
             yield string
 
-    def get_text(self, separator="", strip=False):
+    def get_text(self, separator=u"", strip=False,
+                 types=(NavigableString, CData)):
         """
         Get all child strings, concatenated using the given separator.
         """
-        return separator.join([s for s in self._all_strings(strip)])
+        return separator.join([s for s in self._all_strings(
+                    strip, types=types)])
     getText = get_text
     text = property(get_text)
 
@@ -835,6 +860,7 @@ class Tag(PageElement):
         while i is not None:
             next = i.next_element
             i.__dict__.clear()
+            i.contents = []
             i = next
 
     def clear(self, decompose=False):
@@ -966,6 +992,13 @@ class Tag(PageElement):
         u = self.decode(indent_level, encoding, formatter)
         return u.encode(encoding, errors)
 
+    def _should_pretty_print(self, indent_level):
+        """Should this tag be pretty-printed?"""
+        return (
+            indent_level is not None and
+            (self.name not in HTMLAwareEntitySubstitution.preformatted_tags
+             or self._is_xml))
+
     def decode(self, indent_level=None,
                eventual_encoding=DEFAULT_OUTPUT_ENCODING,
                formatter="minimal"):
@@ -978,6 +1011,12 @@ class Tag(PageElement):
            document contains a <META> tag that mentions the document's
            encoding.
         """
+
+        # First off, turn a string formatter into a function. This
+        # will stop the lookup from happening over and over again.
+        if not callable(formatter):
+            formatter = self._formatter_for_name(formatter)
+
         attrs = []
         if self.attrs:
             for key, val in sorted(self.attrs.items()):
@@ -987,7 +1026,7 @@ class Tag(PageElement):
                     if isinstance(val, list) or isinstance(val, tuple):
                         val = ' '.join(val)
                     elif not isinstance(val, basestring):
-                        val = str(val)
+                        val = unicode(val)
                     elif (
                         isinstance(val, AttributeValueWithCharsetSubstitution)
                         and eventual_encoding is not None):
@@ -995,26 +1034,30 @@ class Tag(PageElement):
 
                     text = self.format_string(val, formatter)
                     decoded = (
-                        str(key) + '='
+                        unicode(key) + '='
                         + EntitySubstitution.quoted_attribute_value(text))
                 attrs.append(decoded)
         close = ''
         closeTag = ''
-        if self.is_empty_element:
-            close = '/'
-        else:
-            closeTag = '</%s>' % self.name
 
         prefix = ''
         if self.prefix:
             prefix = self.prefix + ":"
 
-        pretty_print = (indent_level is not None)
+        if self.is_empty_element:
+            close = '/'
+        else:
+            closeTag = '</%s%s>' % (prefix, self.name)
+
+        pretty_print = self._should_pretty_print(indent_level)
+        space = ''
+        indent_space = ''
+        if indent_level is not None:
+            indent_space = (' ' * (indent_level - 1))
         if pretty_print:
-            space = (' ' * (indent_level - 1))
+            space = indent_space
             indent_contents = indent_level + 1
         else:
-            space = ''
             indent_contents = None
         contents = self.decode_contents(
             indent_contents, eventual_encoding, formatter)
@@ -1027,8 +1070,10 @@ class Tag(PageElement):
             attribute_string = ''
             if attrs:
                 attribute_string = ' ' + ' '.join(attrs)
-            if pretty_print:
-                s.append(space)
+            if indent_level is not None:
+                # Even if this particular tag is not pretty-printed,
+                # we should indent up to the start of the tag.
+                s.append(indent_space)
             s.append('<%s%s%s%s>' % (
                     prefix, self.name, attribute_string, close))
             if pretty_print:
@@ -1039,7 +1084,10 @@ class Tag(PageElement):
             if pretty_print and closeTag:
                 s.append(space)
             s.append(closeTag)
-            if pretty_print and closeTag and self.next_sibling:
+            if indent_level is not None and closeTag and self.next_sibling:
+                # Even if this particular tag is not pretty-printed,
+                # we're now done with the tag, and we should add a
+                # newline if appropriate.
                 s.append("\n")
             s = ''.join(s)
         return s
@@ -1062,6 +1110,11 @@ class Tag(PageElement):
            document contains a <META> tag that mentions the document's
            encoding.
         """
+        # First off, turn a string formatter into a function. This
+        # will stop the lookup from happening over and over again.
+        if not callable(formatter):
+            formatter = self._formatter_for_name(formatter)
+
         pretty_print = (indent_level is not None)
         s = []
         for c in self:
@@ -1071,13 +1124,13 @@ class Tag(PageElement):
             elif isinstance(c, Tag):
                 s.append(c.decode(indent_level, eventual_encoding,
                                   formatter))
-            if text and indent_level:
+            if text and indent_level and not self.name == 'pre':
                 text = text.strip()
             if text:
-                if pretty_print:
+                if pretty_print and not self.name == 'pre':
                     s.append(" " * (indent_level - 1))
                 s.append(text)
-                if pretty_print:
+                if pretty_print and not self.name == 'pre':
                     s.append("\n")
         return ''.join(s)
 
@@ -1120,6 +1173,7 @@ class Tag(PageElement):
         callable that takes a string and returns whether or not the
         string matches for some custom definition of 'matches'. The
         same is true of the tag name."""
+
         generator = self.descendants
         if not recursive:
             generator = self.children
@@ -1143,6 +1197,207 @@ class Tag(PageElement):
             yield current
             current = current.next_element
 
+    # CSS selector code
+
+    _selector_combinators = ['>', '+', '~']
+    _select_debug = False
+    def select(self, selector, _candidate_generator=None):
+        """Perform a CSS selection operation on the current element."""
+        tokens = selector.split()
+        current_context = [self]
+
+        if tokens[-1] in self._selector_combinators:
+            raise ValueError(
+                'Final combinator "%s" is missing an argument.' % tokens[-1])
+        if self._select_debug:
+            print 'Running CSS selector "%s"' % selector
+        for index, token in enumerate(tokens):
+            if self._select_debug:
+                print ' Considering token "%s"' % token
+            recursive_candidate_generator = None
+            tag_name = None
+            if tokens[index-1] in self._selector_combinators:
+                # This token was consumed by the previous combinator. Skip it.
+                if self._select_debug:
+                    print '  Token was consumed by the previous combinator.'
+                continue
+            # Each operation corresponds to a checker function, a rule
+            # for determining whether a candidate matches the
+            # selector. Candidates are generated by the active
+            # iterator.
+            checker = None
+
+            m = self.attribselect_re.match(token)
+            if m is not None:
+                # Attribute selector
+                tag_name, attribute, operator, value = m.groups()
+                checker = self._attribute_checker(operator, attribute, value)
+
+            elif '#' in token:
+                # ID selector
+                tag_name, tag_id = token.split('#', 1)
+                def id_matches(tag):
+                    return tag.get('id', None) == tag_id
+                checker = id_matches
+
+            elif '.' in token:
+                # Class selector
+                tag_name, klass = token.split('.', 1)
+                classes = set(klass.split('.'))
+                def classes_match(candidate):
+                    return classes.issubset(candidate.get('class', []))
+                checker = classes_match
+
+            elif ':' in token:
+                # Pseudo-class
+                tag_name, pseudo = token.split(':', 1)
+                if tag_name == '':
+                    raise ValueError(
+                        "A pseudo-class must be prefixed with a tag name.")
+                pseudo_attributes = re.match('([a-zA-Z\d-]+)\(([a-zA-Z\d]+)\)', pseudo)
+                found = []
+                if pseudo_attributes is not None:
+                    pseudo_type, pseudo_value = pseudo_attributes.groups()
+                    if pseudo_type == 'nth-of-type':
+                        try:
+                            pseudo_value = int(pseudo_value)
+                        except:
+                            raise NotImplementedError(
+                                'Only numeric values are currently supported for the nth-of-type pseudo-class.')
+                        if pseudo_value < 1:
+                            raise ValueError(
+                                'nth-of-type pseudo-class value must be at least 1.')
+                        class Counter(object):
+                            def __init__(self, destination):
+                                self.count = 0
+                                self.destination = destination
+
+                            def nth_child_of_type(self, tag):
+                                self.count += 1
+                                if self.count == self.destination:
+                                    return True
+                                if self.count > self.destination:
+                                    # Stop the generator that's sending us
+                                    # these things.
+                                    raise StopIteration()
+                                return False
+                        checker = Counter(pseudo_value).nth_child_of_type
+                    else:
+                        raise NotImplementedError(
+                            'Only the following pseudo-classes are implemented: nth-of-type.')
+
+            elif token == '*':
+                # Star selector -- matches everything
+                pass
+            elif token == '>':
+                # Run the next token as a CSS selector against the
+                # direct children of each tag in the current context.
+                recursive_candidate_generator = lambda tag: tag.children
+            elif token == '~':
+                # Run the next token as a CSS selector against the
+                # siblings of each tag in the current context.
+                recursive_candidate_generator = lambda tag: tag.next_siblings
+            elif token == '+':
+                # For each tag in the current context, run the next
+                # token as a CSS selector against the tag's next
+                # sibling that's a tag.
+                def next_tag_sibling(tag):
+                    yield tag.find_next_sibling(True)
+                recursive_candidate_generator = next_tag_sibling
+
+            elif self.tag_name_re.match(token):
+                # Just a tag name.
+                tag_name = token
+            else:
+                raise ValueError(
+                    'Unsupported or invalid CSS selector: "%s"' % token)
+
+            if recursive_candidate_generator:
+                # This happens when the selector looks like  "> foo".
+                #
+                # The generator calls select() recursively on every
+                # member of the current context, passing in a different
+                # candidate generator and a different selector.
+                #
+                # In the case of "> foo", the candidate generator is
+                # one that yields a tag's direct children (">"), and
+                # the selector is "foo".
+                next_token = tokens[index+1]
+                def recursive_select(tag):
+                    if self._select_debug:
+                        print '    Calling select("%s") recursively on %s %s' % (next_token, tag.name, tag.attrs)
+                        print '-' * 40
+                    for i in tag.select(next_token, recursive_candidate_generator):
+                        if self._select_debug:
+                            print '(Recursive select picked up candidate %s %s)' % (i.name, i.attrs)
+                        yield i
+                    if self._select_debug:
+                        print '-' * 40
+                _use_candidate_generator = recursive_select
+            elif _candidate_generator is None:
+                # By default, a tag's candidates are all of its
+                # children. If tag_name is defined, only yield tags
+                # with that name.
+                if self._select_debug:
+                    if tag_name:
+                        check = "[any]"
+                    else:
+                        check = tag_name
+                    print '   Default candidate generator, tag name="%s"' % check
+                if self._select_debug:
+                    # This is redundant with later code, but it stops
+                    # a bunch of bogus tags from cluttering up the
+                    # debug log.
+                    def default_candidate_generator(tag):
+                        for child in tag.descendants:
+                            if not isinstance(child, Tag):
+                                continue
+                            if tag_name and not child.name == tag_name:
+                                continue
+                            yield child
+                    _use_candidate_generator = default_candidate_generator
+                else:
+                    _use_candidate_generator = lambda tag: tag.descendants
+            else:
+                _use_candidate_generator = _candidate_generator
+
+            new_context = []
+            new_context_ids = set([])
+            for tag in current_context:
+                if self._select_debug:
+                    print "    Running candidate generator on %s %s" % (
+                        tag.name, repr(tag.attrs))
+                for candidate in _use_candidate_generator(tag):
+                    if not isinstance(candidate, Tag):
+                        continue
+                    if tag_name and candidate.name != tag_name:
+                        continue
+                    if checker is not None:
+                        try:
+                            result = checker(candidate)
+                        except StopIteration:
+                            # The checker has decided we should no longer
+                            # run the generator.
+                            break
+                    if checker is None or result:
+                        if self._select_debug:
+                            print "     SUCCESS %s %s" % (candidate.name, repr(candidate.attrs))
+                        if id(candidate) not in new_context_ids:
+                            # If a tag matches a selector more than once,
+                            # don't include it in the context more than once.
+                            new_context.append(candidate)
+                            new_context_ids.add(id(candidate))
+                    elif self._select_debug:
+                        print "     FAILURE %s %s" % (candidate.name, repr(candidate.attrs))
+
+            current_context = new_context
+
+        if self._select_debug:
+            print "Final verdict:"
+            for i in current_context:
+                print " %s %s" % (i.name, i.attrs)
+        return current_context
+
     # Old names for backwards compatibility
     def childGenerator(self):
         return self.children
@@ -1150,10 +1405,13 @@ class Tag(PageElement):
     def recursiveChildGenerator(self):
         return self.descendants
 
-    # This was kind of misleading because has_key() (attributes) was
-    # different from __in__ (contents). has_key() is gone in Python 3,
-    # anyway.
-    has_key = has_attr
+    def has_key(self, key):
+        """This was kind of misleading because has_key() (attributes)
+        was different from __in__ (contents). has_key() is gone in
+        Python 3, anyway."""
+        warnings.warn('has_key is deprecated. Use has_attr("%s") instead.' % (
+                key))
+        return self.has_attr(key)
 
 # Next, a couple classes to represent queries and their results.
 class SoupStrainer(object):
@@ -1168,6 +1426,12 @@ class SoupStrainer(object):
             kwargs['class'] = attrs
             attrs = None
 
+        if 'class_' in kwargs:
+            # Treat class_="foo" as a search for the 'class'
+            # attribute, overriding any non-dict value for attrs.
+            kwargs['class'] = kwargs['class_']
+            del kwargs['class_']
+
         if kwargs:
             if attrs:
                 attrs = attrs.copy()
@@ -1342,6 +1606,6 @@ class SoupStrainer(object):
 class ResultSet(list):
     """A ResultSet is just a list that keeps track of the SoupStrainer
     that created it."""
-    def __init__(self, source):
-        list.__init__([])
+    def __init__(self, source, result=()):
+        super(ResultSet, self).__init__(result)
         self.source = source

libs/bs4/testing.py

@@ -81,6 +81,11 @@ class HTMLTreeBuilderSmokeTest(object):
         self.assertDoctypeHandled(
             'html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"')
 
+    def test_empty_doctype(self):
+        soup = self.soup("<!DOCTYPE>")
+        doctype = soup.contents[0]
+        self.assertEqual("", doctype.strip())
+
     def test_public_doctype_with_url(self):
         doctype = 'html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"'
         self.assertDoctypeHandled(doctype)
@@ -159,6 +164,12 @@ class HTMLTreeBuilderSmokeTest(object):
         comment = soup.find(text="foobar")
         self.assertEqual(comment.__class__, Comment)
 
+        # The comment is properly integrated into the tree.
+        foo = soup.find(text="foo")
+        self.assertEqual(comment, foo.next_element)
+        baz = soup.find(text="baz")
+        self.assertEqual(comment, baz.previous_element)
+
     def test_preserved_whitespace_in_pre_and_textarea(self):
         """Whitespace must be preserved in <pre> and <textarea> tags."""
         self.assertSoupEquals("<pre>   </pre>")
@@ -202,6 +213,14 @@ class HTMLTreeBuilderSmokeTest(object):
             "<tbody><tr><td>Bar</td></tr></tbody>"
             "<tfoot><tr><td>Baz</td></tr></tfoot></table>")
 
+    def test_deeply_nested_multivalued_attribute(self):
+        # html5lib can set the attributes of the same tag many times
+        # as it rearranges the tree. This has caused problems with
+        # multivalued attributes.
+        markup = '<table><div><div class="css"></div></div></table>'
+        soup = self.soup(markup)
+        self.assertEqual(["css"], soup.div.div['class'])
+
     def test_angle_brackets_in_attribute_values_are_escaped(self):
         self.assertSoupEquals('<a b="<a>"></a>', '<a b="&lt;a&gt;"></a>')
 
@@ -209,12 +228,14 @@ class HTMLTreeBuilderSmokeTest(object):
         expect = u'<p id="pi\N{LATIN SMALL LETTER N WITH TILDE}ata"></p>'
         self.assertSoupEquals('<p id="pi&#241;ata"></p>', expect)
         self.assertSoupEquals('<p id="pi&#xf1;ata"></p>', expect)
+        self.assertSoupEquals('<p id="pi&#Xf1;ata"></p>', expect)
         self.assertSoupEquals('<p id="pi&ntilde;ata"></p>', expect)
 
     def test_entities_in_text_converted_to_unicode(self):
         expect = u'<p>pi\N{LATIN SMALL LETTER N WITH TILDE}ata</p>'
         self.assertSoupEquals("<p>pi&#241;ata</p>", expect)
         self.assertSoupEquals("<p>pi&#xf1;ata</p>", expect)
+        self.assertSoupEquals("<p>pi&#Xf1;ata</p>", expect)
         self.assertSoupEquals("<p>pi&ntilde;ata</p>", expect)
 
     def test_quot_entity_converted_to_quotation_mark(self):
@@ -227,6 +248,12 @@ class HTMLTreeBuilderSmokeTest(object):
         self.assertSoupEquals("&#x10000000000000;", expect)
         self.assertSoupEquals("&#1000000000;", expect)
 
+    def test_multipart_strings(self):
+        "Mostly to prevent a recurrence of a bug in the html5lib treebuilder."
+        soup = self.soup("<html><h2>\nfoo</h2><p></p></html>")
+        self.assertEqual("p", soup.h2.string.next_element.name)
+        self.assertEqual("p", soup.p.name)
+
     def test_basic_namespaces(self):
         """Parsers don't need to *understand* namespaces, but at the
         very least they should not choke on namespaces or lose
@@ -254,6 +281,14 @@ class HTMLTreeBuilderSmokeTest(object):
     # to detect any differences between them.
     #
 
+    def test_can_parse_unicode_document(self):
+        # A seemingly innocuous document... but it's in Unicode! And
+        # it contains characters that can't be represented in the
+        # encoding found in the  declaration! The horror!
+        markup = u'<html><head><meta encoding="euc-jp"></head><body>Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!</body>'
+        soup = self.soup(markup)
+        self.assertEqual(u'Sacr\xe9 bleu!', soup.body.string)
+
     def test_soupstrainer(self):
         """Parsers should be able to work with SoupStrainers."""
         strainer = SoupStrainer("b")
@@ -445,6 +480,28 @@ class XMLTreeBuilderSmokeTest(object):
         self.assertEqual(
             soup.encode("utf-8"), markup)
 
+    def test_formatter_processes_script_tag_for_xml_documents(self):
+        doc = """
+  <script type="text/javascript">
+  </script>
+"""
+        soup = BeautifulSoup(doc, "xml")
+        # lxml would have stripped this while parsing, but we can add
+        # it later.
+        soup.script.string = 'console.log("< < hey > > ");'
+        encoded = soup.encode()
+        self.assertTrue(b"&lt; &lt; hey &gt; &gt;" in encoded)
+
+    def test_can_parse_unicode_document(self):
+        markup = u'<?xml version="1.0" encoding="euc-jp"><root>Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!</root>'
+        soup = self.soup(markup)
+        self.assertEqual(u'Sacr\xe9 bleu!', soup.root.string)
+
+    def test_popping_namespaced_tag(self):
+        markup = '<rss xmlns:dc="foo"><dc:creator>b</dc:creator><dc:date>2012-07-02T20:33:42Z</dc:date><dc:rights>c</dc:rights><image>d</image></rss>'
+        soup = self.soup(markup)
+        self.assertEqual(
+            unicode(soup.rss), markup)
 
     def test_docstring_includes_correct_encoding(self):
         soup = self.soup("<root/>")
@@ -472,6 +529,20 @@ class XMLTreeBuilderSmokeTest(object):
         self.assertEqual("http://example.com/", root['xmlns:a'])
         self.assertEqual("http://example.net/", root['xmlns:b'])
 
+    def test_closing_namespaced_tag(self):
+        markup = '<p xmlns:dc="http://purl.org/dc/elements/1.1/"><dc:date>20010504</dc:date></p>'
+        soup = self.soup(markup)
+        self.assertEqual(unicode(soup.p), markup)
+
+    def test_namespaced_attributes(self):
+        markup = '<foo xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"><bar xsi:schemaLocation="http://www.example.com"/></foo>'
+        soup = self.soup(markup)
+        self.assertEqual(unicode(soup.foo), markup)
+
+    def test_namespaced_attributes_xml_namespace(self):
+        markup = '<foo xml:lang="fr">bar</foo>'
+        soup = self.soup(markup)
+        self.assertEqual(unicode(soup.foo), markup)
 
 class HTML5TreeBuilderSmokeTest(HTMLTreeBuilderSmokeTest):
     """Smoke test for a tree builder that supports HTML5."""
@@ -501,6 +572,12 @@ class HTML5TreeBuilderSmokeTest(HTMLTreeBuilderSmokeTest):
         self.assertEqual(namespace, soup.math.namespace)
         self.assertEqual(namespace, soup.msqrt.namespace)
 
+    def test_xml_declaration_becomes_comment(self):
+        markup = '<?xml version="1.0" encoding="utf-8"?><html></html>'
+        soup = self.soup(markup)
+        self.assertTrue(isinstance(soup.contents[0], Comment))
+        self.assertEqual(soup.contents[0], '?xml version="1.0" encoding="utf-8"?')
+        self.assertEqual("html", soup.contents[0].next_element.name)
 
 def skipIf(condition, reason):
    def nothing(test, *args, **kwargs):

libs/gntp/__init__.py

@@ -1,509 +0,0 @@
-import re
-import hashlib
-import time
-import StringIO
-
-__version__ = '0.8'
-
-#GNTP/<version> <messagetype> <encryptionAlgorithmID>[:<ivValue>][ <keyHashAlgorithmID>:<keyHash>.<salt>]
-GNTP_INFO_LINE = re.compile(
-	'GNTP/(?P<version>\d+\.\d+) (?P<messagetype>REGISTER|NOTIFY|SUBSCRIBE|\-OK|\-ERROR)' +
-	' (?P<encryptionAlgorithmID>[A-Z0-9]+(:(?P<ivValue>[A-F0-9]+))?) ?' +
-	'((?P<keyHashAlgorithmID>[A-Z0-9]+):(?P<keyHash>[A-F0-9]+).(?P<salt>[A-F0-9]+))?\r\n',
-	re.IGNORECASE
-)
-
-GNTP_INFO_LINE_SHORT = re.compile(
-	'GNTP/(?P<version>\d+\.\d+) (?P<messagetype>REGISTER|NOTIFY|SUBSCRIBE|\-OK|\-ERROR)',
-	re.IGNORECASE
-)
-
-GNTP_HEADER = re.compile('([\w-]+):(.+)')
-
-GNTP_EOL = '\r\n'
-
-
-class BaseError(Exception):
-	def gntp_error(self):
-		error = GNTPError(self.errorcode, self.errordesc)
-		return error.encode()
-
-
-class ParseError(BaseError):
-	errorcode = 500
-	errordesc = 'Error parsing the message'
-
-
-class AuthError(BaseError):
-	errorcode = 400
-	errordesc = 'Error with authorization'
-
-
-class UnsupportedError(BaseError):
-	errorcode = 500
-	errordesc = 'Currently unsupported by gntp.py'
-
-
-class _GNTPBuffer(StringIO.StringIO):
-	"""GNTP Buffer class"""
-	def writefmt(self, message = "", *args):
-		"""Shortcut function for writing GNTP Headers"""
-		self.write((message % args).encode('utf8', 'replace'))
-		self.write(GNTP_EOL)
-
-
-class _GNTPBase(object):
-	"""Base initilization
-
-	:param string messagetype: GNTP Message type
-	:param string version: GNTP Protocol version
-	:param string encription: Encryption protocol
-	"""
-	def __init__(self, messagetype = None, version = '1.0', encryption = None):
-		self.info = {
-			'version': version,
-			'messagetype': messagetype,
-			'encryptionAlgorithmID': encryption
-		}
-		self.headers = {}
-		self.resources = {}
-
-	def __str__(self):
-		return self.encode()
-
-	def _parse_info(self, data):
-		"""Parse the first line of a GNTP message to get security and other info values
-
-		:param string data: GNTP Message
-		:return dict: Parsed GNTP Info line
-		"""
-
-		match = GNTP_INFO_LINE.match(data)
-
-		if not match:
-			raise ParseError('ERROR_PARSING_INFO_LINE')
-
-		info = match.groupdict()
-		if info['encryptionAlgorithmID'] == 'NONE':
-			info['encryptionAlgorithmID'] = None
-
-		return info
-
-	def set_password(self, password, encryptAlgo = 'MD5'):
-		"""Set a password for a GNTP Message
-
-		:param string password: Null to clear password
-		:param string encryptAlgo: Supports MD5, SHA1, SHA256, SHA512
-		"""
-		hash = {
-			'MD5': hashlib.md5,
-			'SHA1': hashlib.sha1,
-			'SHA256': hashlib.sha256,
-			'SHA512': hashlib.sha512,
-		}
-
-		self.password = password
-		self.encryptAlgo = encryptAlgo.upper()
-		if not password:
-			self.info['encryptionAlgorithmID'] = None
-			self.info['keyHashAlgorithm'] = None
-			return
-		if not self.encryptAlgo in hash.keys():
-			raise UnsupportedError('INVALID HASH "%s"' % self.encryptAlgo)
-
-		hashfunction = hash.get(self.encryptAlgo)
-
-		password = password.encode('utf8')
-		seed = time.ctime()
-		salt = hashfunction(seed).hexdigest()
-		saltHash = hashfunction(seed).digest()
-		keyBasis = password + saltHash
-		key = hashfunction(keyBasis).digest()
-		keyHash = hashfunction(key).hexdigest()
-
-		self.info['keyHashAlgorithmID'] = self.encryptAlgo
-		self.info['keyHash'] = keyHash.upper()
-		self.info['salt'] = salt.upper()
-
-	def _decode_hex(self, value):
-		"""Helper function to decode hex string to `proper` hex string
-
-		:param string value: Human readable hex string
-		:return string: Hex string
-		"""
-		result = ''
-		for i in range(0, len(value), 2):
-			tmp = int(value[i:i + 2], 16)
-			result += chr(tmp)
-		return result
-
-	def _decode_binary(self, rawIdentifier, identifier):
-		rawIdentifier += '\r\n\r\n'
-		dataLength = int(identifier['Length'])
-		pointerStart = self.raw.find(rawIdentifier) + len(rawIdentifier)
-		pointerEnd = pointerStart + dataLength
-		data = self.raw[pointerStart:pointerEnd]
-		if not len(data) == dataLength:
-			raise ParseError('INVALID_DATA_LENGTH Expected: %s Recieved %s' % (dataLength, len(data)))
-		return data
-
-	def _validate_password(self, password):
-		"""Validate GNTP Message against stored password"""
-		self.password = password
-		if password == None:
-			raise AuthError('Missing password')
-		keyHash = self.info.get('keyHash', None)
-		if keyHash is None and self.password is None:
-			return True
-		if keyHash is None:
-			raise AuthError('Invalid keyHash')
-		if self.password is None:
-			raise AuthError('Missing password')
-
-		password = self.password.encode('utf8')
-		saltHash = self._decode_hex(self.info['salt'])
-
-		keyBasis = password + saltHash
-		key = hashlib.md5(keyBasis).digest()
-		keyHash = hashlib.md5(key).hexdigest()
-
-		if not keyHash.upper() == self.info['keyHash'].upper():
-			raise AuthError('Invalid Hash')
-		return True
-
-	def validate(self):
-		"""Verify required headers"""
-		for header in self._requiredHeaders:
-			if not self.headers.get(header, False):
-				raise ParseError('Missing Notification Header: ' + header)
-
-	def _format_info(self):
-		"""Generate info line for GNTP Message
-
-		:return string:
-		"""
-		info = u'GNTP/%s %s' % (
-			self.info.get('version'),
-			self.info.get('messagetype'),
-		)
-		if self.info.get('encryptionAlgorithmID', None):
-			info += ' %s:%s' % (
-				self.info.get('encryptionAlgorithmID'),
-				self.info.get('ivValue'),
-			)
-		else:
-			info += ' NONE'
-
-		if self.info.get('keyHashAlgorithmID', None):
-			info += ' %s:%s.%s' % (
-				self.info.get('keyHashAlgorithmID'),
-				self.info.get('keyHash'),
-				self.info.get('salt')
-			)
-
-		return info
-
-	def _parse_dict(self, data):
-		"""Helper function to parse blocks of GNTP headers into a dictionary
-
-		:param string data:
-		:return dict:
-		"""
-		dict = {}
-		for line in data.split('\r\n'):
-			match = GNTP_HEADER.match(line)
-			if not match:
-				continue
-
-			key = unicode(match.group(1).strip(), 'utf8', 'replace')
-			val = unicode(match.group(2).strip(), 'utf8', 'replace')
-			dict[key] = val
-		return dict
-
-	def add_header(self, key, value):
-		if isinstance(value, unicode):
-			self.headers[key] = value
-		else:
-			self.headers[key] = unicode('%s' % value, 'utf8', 'replace')
-
-	def add_resource(self, data):
-		"""Add binary resource
-
-		:param string data: Binary Data
-		"""
-		identifier = hashlib.md5(data).hexdigest()
-		self.resources[identifier] = data
-		return 'x-growl-resource://%s' % identifier
-
-	def decode(self, data, password = None):
-		"""Decode GNTP Message
-
-		:param string data:
-		"""
-		self.password = password
-		self.raw = data
-		parts = self.raw.split('\r\n\r\n')
-		self.info = self._parse_info(data)
-		self.headers = self._parse_dict(parts[0])
-
-	def encode(self):
-		"""Encode a generic GNTP Message
-
-		:return string: GNTP Message ready to be sent
-		"""
-
-		buffer = _GNTPBuffer()
-
-		buffer.writefmt(self._format_info())
-
-		#Headers
-		for k, v in self.headers.iteritems():
-			buffer.writefmt('%s: %s', k, v)
-		buffer.writefmt()
-
-		#Resources
-		for resource, data in self.resources.iteritems():
-			buffer.writefmt('Identifier: %s', resource)
-			buffer.writefmt('Length: %d', len(data))
-			buffer.writefmt()
-			buffer.write(data)
-			buffer.writefmt()
-			buffer.writefmt()
-
-		return buffer.getvalue()
-
-
-class GNTPRegister(_GNTPBase):
-	"""Represents a GNTP Registration Command
-
-	:param string data: (Optional) See decode()
-	:param string password: (Optional) Password to use while encoding/decoding messages
-	"""
-	_requiredHeaders = [
-		'Application-Name',
-		'Notifications-Count'
-	]
-	_requiredNotificationHeaders = ['Notification-Name']
-
-	def __init__(self, data = None, password = None):
-		_GNTPBase.__init__(self, 'REGISTER')
-		self.notifications = []
-
-		if data:
-			self.decode(data, password)
-		else:
-			self.set_password(password)
-			self.add_header('Application-Name', 'pygntp')
-			self.add_header('Notifications-Count', 0)
-
-	def validate(self):
-		'''Validate required headers and validate notification headers'''
-		for header in self._requiredHeaders:
-			if not self.headers.get(header, False):
-				raise ParseError('Missing Registration Header: ' + header)
-		for notice in self.notifications:
-			for header in self._requiredNotificationHeaders:
-				if not notice.get(header, False):
-					raise ParseError('Missing Notification Header: ' + header)
-
-	def decode(self, data, password):
-		"""Decode existing GNTP Registration message
-
-		:param string data: Message to decode
-		"""
-		self.raw = data
-		parts = self.raw.split('\r\n\r\n')
-		self.info = self._parse_info(data)
-		self._validate_password(password)
-		self.headers = self._parse_dict(parts[0])
-
-		for i, part in enumerate(parts):
-			if i == 0:
-				continue # Skip Header
-			if part.strip() == '':
-				continue
-			notice = self._parse_dict(part)
-			if notice.get('Notification-Name', False):
-				self.notifications.append(notice)
-			elif notice.get('Identifier', False):
-				notice['Data'] = self._decode_binary(part, notice)
-				#open('register.png','wblol').write(notice['Data'])
-				self.resources[notice.get('Identifier')] = notice
-
-	def add_notification(self, name, enabled = True):
-		"""Add new Notification to Registration message
-
-		:param string name: Notification Name
-		:param boolean enabled: Enable this notification by default
-		"""
-		notice = {}
-		notice['Notification-Name'] = u'%s' % name
-		notice['Notification-Enabled'] = u'%s' % enabled
-
-		self.notifications.append(notice)
-		self.add_header('Notifications-Count', len(self.notifications))
-
-	def encode(self):
-		"""Encode a GNTP Registration Message
-
-		:return string: Encoded GNTP Registration message
-		"""
-
-		buffer = _GNTPBuffer()
-
-		buffer.writefmt(self._format_info())
-
-		#Headers
-		for k, v in self.headers.iteritems():
-			buffer.writefmt('%s: %s', k, v)
-		buffer.writefmt()
-
-		#Notifications
-		if len(self.notifications) > 0:
-			for notice in self.notifications:
-				for k, v in notice.iteritems():
-					buffer.writefmt('%s: %s', k, v)
-				buffer.writefmt()
-
-		#Resources
-		for resource, data in self.resources.iteritems():
-			buffer.writefmt('Identifier: %s', resource)
-			buffer.writefmt('Length: %d', len(data))
-			buffer.writefmt()
-			buffer.write(data)
-			buffer.writefmt()
-			buffer.writefmt()
-
-		return buffer.getvalue()
-
-
-class GNTPNotice(_GNTPBase):
-	"""Represents a GNTP Notification Command
-
-	:param string data: (Optional) See decode()
-	:param string app: (Optional) Set Application-Name
-	:param string name: (Optional) Set Notification-Name
-	:param string title: (Optional) Set Notification Title
-	:param string password: (Optional) Password to use while encoding/decoding messages
-	"""
-	_requiredHeaders = [
-		'Application-Name',
-		'Notification-Name',
-		'Notification-Title'
-	]
-
-	def __init__(self, data = None, app = None, name = None, title = None, password = None):
-		_GNTPBase.__init__(self, 'NOTIFY')
-
-		if data:
-			self.decode(data, password)
-		else:
-			self.set_password(password)
-			if app:
-				self.add_header('Application-Name', app)
-			if name:
-				self.add_header('Notification-Name', name)
-			if title:
-				self.add_header('Notification-Title', title)
-
-	def decode(self, data, password):
-		"""Decode existing GNTP Notification message
-
-		:param string data: Message to decode.
-		"""
-		self.raw = data
-		parts = self.raw.split('\r\n\r\n')
-		self.info = self._parse_info(data)
-		self._validate_password(password)
-		self.headers = self._parse_dict(parts[0])
-
-		for i, part in enumerate(parts):
-			if i == 0:
-				continue # Skip Header
-			if part.strip() == '':
-				continue
-			notice = self._parse_dict(part)
-			if notice.get('Identifier', False):
-				notice['Data'] = self._decode_binary(part, notice)
-				#open('notice.png','wblol').write(notice['Data'])
-				self.resources[notice.get('Identifier')] = notice
-
-
-class GNTPSubscribe(_GNTPBase):
-	"""Represents a GNTP Subscribe Command
-
-	:param string data: (Optional) See decode()
-	:param string password: (Optional) Password to use while encoding/decoding messages
-	"""
-	_requiredHeaders = [
-		'Subscriber-ID',
-		'Subscriber-Name',
-	]
-
-	def __init__(self, data = None, password = None):
-		_GNTPBase.__init__(self, 'SUBSCRIBE')
-		if data:
-			self.decode(data, password)
-		else:
-			self.set_password(password)
-
-
-class GNTPOK(_GNTPBase):
-	"""Represents a GNTP OK Response
-
-	:param string data: (Optional) See _GNTPResponse.decode()
-	:param string action: (Optional) Set type of action the OK Response is for
-	"""
-	_requiredHeaders = ['Response-Action']
-
-	def __init__(self, data = None, action = None):
-		_GNTPBase.__init__(self, '-OK')
-		if data:
-			self.decode(data)
-		if action:
-			self.add_header('Response-Action', action)
-
-
-class GNTPError(_GNTPBase):
-	"""Represents a GNTP Error response
-
-	:param string data: (Optional) See _GNTPResponse.decode()
-	:param string errorcode: (Optional) Error code
-	:param string errordesc: (Optional) Error Description
-	"""
-	_requiredHeaders = ['Error-Code', 'Error-Description']
-
-	def __init__(self, data = None, errorcode = None, errordesc = None):
-		_GNTPBase.__init__(self, '-ERROR')
-		if data:
-			self.decode(data)
-		if errorcode:
-			self.add_header('Error-Code', errorcode)
-			self.add_header('Error-Description', errordesc)
-
-	def error(self):
-		return (self.headers.get('Error-Code', None),
-			self.headers.get('Error-Description', None))
-
-
-def parse_gntp(data, password = None):
-	"""Attempt to parse a message as a GNTP message
-
-	:param string data: Message to be parsed
-	:param string password: Optional password to be used to verify the message
-	"""
-	match = GNTP_INFO_LINE_SHORT.match(data)
-	if not match:
-		raise ParseError('INVALID_GNTP_INFO')
-	info = match.groupdict()
-	if info['messagetype'] == 'REGISTER':
-		return GNTPRegister(data, password = password)
-	elif info['messagetype'] == 'NOTIFY':
-		return GNTPNotice(data, password = password)
-	elif info['messagetype'] == 'SUBSCRIBE':
-		return GNTPSubscribe(data, password = password)
-	elif info['messagetype'] == '-OK':
-		return GNTPOK(data)
-	elif info['messagetype'] == '-ERROR':
-		return GNTPError(data)
-	raise ParseError('INVALID_GNTP_MESSAGE')