Py3 fixes

close #3376

CouchPotato.py

@@ -10,7 +10,6 @@ import socket
 import subprocess
 import sys
 import traceback
-import time
 
 # Root path
 base_path = dirname(os.path.abspath(__file__))
@@ -19,7 +18,12 @@ base_path = dirname(os.path.abspath(__file__))
 sys.path.insert(0, os.path.join(base_path, 'libs'))
 
 from couchpotato.environment import Env
-from couchpotato.core.helpers.variable import getDataDir
+from couchpotato.core.helpers.variable import getDataDir, removePyc
+
+
+# Remove pyc files before dynamic load (sees .pyc files regular .py modules)
+removePyc(base_path)
+
 
 class Loader(object):
 
@@ -29,7 +33,7 @@ class Loader(object):
 
         # Get options via arg
         from couchpotato.runner import getOptions
-        self.options = getOptions(base_path, sys.argv[1:])
+        self.options = getOptions(sys.argv[1:])
 
         # Load settings
         settings = Env.get('settings')
@@ -50,14 +54,14 @@ class Loader(object):
         # Create logging dir
         self.log_dir = os.path.join(self.data_dir, 'logs');
         if not os.path.isdir(self.log_dir):
-            os.mkdir(self.log_dir)
+            os.makedirs(self.log_dir)
 
         # Logging
         from couchpotato.core.logger import CPLog
         self.log = CPLog(__name__)
 
         formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s', '%H:%M:%S')
-        hdlr = handlers.RotatingFileHandler(os.path.join(self.log_dir, 'error.log'), 'a', 500000, 10)
+        hdlr = handlers.RotatingFileHandler(os.path.join(self.log_dir, 'error.log'), 'a', 500000, 10, encoding = 'utf-8')
         hdlr.setLevel(logging.CRITICAL)
         hdlr.setFormatter(formatter)
         self.log.logger.addHandler(hdlr)
@@ -67,10 +71,11 @@ class Loader(object):
         signal.signal(signal.SIGTERM, lambda signum, stack_frame: sys.exit(1))
 
         from couchpotato.core.event import addEvent
-        addEvent('app.after_shutdown', self.afterShutdown)
+        addEvent('app.do_shutdown', self.setRestart)
 
-    def afterShutdown(self, restart):
+    def setRestart(self, restart):
         self.do_restart = restart
+        return True
 
     def onExit(self, signal, frame):
         from couchpotato.core.event import fireEvent
@@ -98,7 +103,6 @@ class Loader(object):
 
             # Release log files and shutdown logger
             logging.shutdown()
-            time.sleep(3)
 
             args = [sys.executable] + [os.path.join(base_path, os.path.basename(__file__))] + sys.argv[1:]
             subprocess.Popen(args)

README.md

@@ -17,9 +17,9 @@ Windows, see [the CP forum](http://couchpota.to/forum/showthread.php?tid=14) for
 * Open up `Git Bash` (or CMD) and go to the folder you want to install CP. Something like Program Files.
 * Run `git clone https://github.com/RuudBurger/CouchPotatoServer.git`.
 * You can now start CP via `CouchPotatoServer\CouchPotato.py` to start
-* Your browser should open up, but if it doesn't go to: `http://localhost:5050/`
+* Your browser should open up, but if it doesn't go to `http://localhost:5050/`
 
-OSx:
+OS X:
 
 * If you're on Leopard (10.5) install Python 2.6+: [Python 2.6.5](http://www.python.org/download/releases/2.6.5/)
 * Install [GIT](http://git-scm.com/)
@@ -27,20 +27,27 @@ OSx:
 * Go to your App folder `cd /Applications`
 * Run `git clone https://github.com/RuudBurger/CouchPotatoServer.git`
 * Then do `python CouchPotatoServer/CouchPotato.py`
-* Your browser should open up, but if it doesn't go to: `http://localhost:5050/`
+* Your browser should open up, but if it doesn't go to `http://localhost:5050/`
 
-Linux (ubuntu / debian):
+Linux:
 
-* Install [GIT](http://git-scm.com/) with `apt-get install git-core`
+* (Ubuntu / Debian) Install [GIT](http://git-scm.com/) with `apt-get install git-core`
+* (Fedora / CentOS) Install [GIT](http://git-scm.com/) with `yum install git`
 * 'cd' to the folder of your choosing.
 * Run `git clone https://github.com/RuudBurger/CouchPotatoServer.git`
 * Then do `python CouchPotatoServer/CouchPotato.py` to start
-* To run on boot copy the init script. `sudo cp CouchPotatoServer/init/ubuntu /etc/init.d/couchpotato`
-* Change the paths inside the init script. `sudo nano /etc/init.d/couchpotato`
-* Make it executable. `sudo chmod +x /etc/init.d/couchpotato`
-* Add it to defaults. `sudo update-rc.d couchpotato defaults`
-* Open your browser and go to: `http://localhost:5050/`
+* (Ubuntu / Debian) To run on boot copy the init script `sudo cp CouchPotatoServer/init/ubuntu /etc/init.d/couchpotato`
+* (Ubuntu / Debian) Copy the default paths file `sudo cp CouchPotatoServer/init/ubuntu.default /etc/default/couchpotato`
+* (Ubuntu / Debian) Change the paths inside the default file `sudo nano /etc/default/couchpotato`
+* (Ubuntu / Debian) Make it executable `sudo chmod +x /etc/init.d/couchpotato`
+* (Ubuntu / Debian) Add it to defaults `sudo update-rc.d couchpotato defaults`
+* (systemd) To run on boot copy the systemd config `sudo cp CouchPotatoServer/init/couchpotato.fedora.service /etc/systemd/system/couchpotato.service`
+* (systemd) Update the systemd config file with your user and path to CouchPotato.py 
+* (systemd) Enable it at boot with `sudo systemctl enable couchpotato`
+* Open your browser and go to `http://localhost:5050/`
 
+Docker:
+* You can use [razorgirl's Dockerfile](https://github.com/razorgirl/docker-couchpotato) to quickly build your own isolated app container. It's based on the Linux instructions above. For more info about Docker check out the [official website](https://www.docker.com).
 
 FreeBSD :
 

contributing.md

@@ -13,6 +13,8 @@ Lastly, for anything related to CouchPotato, feel free to stop by the [forum](ht
 ## Issues
 Issues are intended for reporting bugs and weird behaviour or suggesting improvements to CouchPotatoServer.
 Before you submit an issue, please go through the following checklist:
+ * **FILL IN ALL THE FIELDS ASKED FOR**
+ * **POST MORE THAN A SINGLE LINE LOG**, if you do, you'd better have a easy reproducable bug
  * Search through existing issues (*including closed issues!*) first: you might be able to get your answer there.
  * Double check your issue manually, because it could be an external issue. 
  * Post logs with your issue: Without seeing what is going on, the developers can't reproduce the error.
@@ -22,15 +24,17 @@ Before you submit an issue, please go through the following checklist:
  * What providers are you using? (While your logs include these, scanning through hundreds of lines of logs isn't our hobby)
  * Post the logs from the *config* directory, please do not copy paste the UI. Use pastebin to store these logs!
  * Give a short step by step of how to reproduce the error.
- * What hardware / OS are you using and what are its limitations? For example: NAS can be slow and maybe have a different version of python installed then when you use CP on OSX or Windows.
+ * What hardware / OS are you using and what are its limitations? For example: NAS can be slow and maybe have a different version of python installed than when you use CP on OS X or Windows.
  * Your issue might be marked with the "can't reproduce" tag. Don't ask why your issue was closed if it says so in the tag.
- * If you're running on a NAS (QNAP, Austor etc..) with pre-made packages, make sure these are set up to use our source repository (RuudBurger/CouchPotatoServer) and nothing else!!
- 
-The more relevant information you can provide, the more likely it is the issue will be resolved rather than closed.
+ * If you're running on a NAS (QNAP, Austor, Synology etc.) with pre-made packages, make sure these are set up to use our source repository (RuudBurger/CouchPotatoServer) and nothing else!
+ * Do not "bump" issues with "Any updates on this" or whatever. Yes I've seen it, you don't have to remind me of it. There will be an update when the code is done or I need information. If you feel the need to do so, you'd better have more info on the issue.
+
+The more relevant information you provide, the more likely that your issue will be resolved.
+If you don't follow any of the checks above, I'll close the issue. If you are wondering why (and ask) I'll block you from posting new issues and the repo.
 
 ## Pull Requests
 Pull requests are intended for contributing code or documentation to the project. Before you submit a pull request, consider the following:
  * Make sure your pull request is made for the *develop* branch (or relevant feature branch).
  * Have you tested your PR? If not, why?
- * Does your PR have any limitations we should know of?
+ * Does your PR have any limitations I should know of?
  * Is your PR up-to-date with the branch you're trying to push into?

couchpotato/__init__.py

@@ -1,3 +1,7 @@
+import os
+import time
+import traceback
+
 from couchpotato.api import api_docs, api_docs_missing, api
 from couchpotato.core.event import fireEvent
 from couchpotato.core.helpers.variable import md5, tryInt
@@ -5,9 +9,6 @@ from couchpotato.core.logger import CPLog
 from couchpotato.environment import Env
 from tornado import template
 from tornado.web import RequestHandler, authenticated
-import os
-import time
-import traceback
 
 
 log = CPLog(__name__)
@@ -45,7 +46,7 @@ class WebHandler(BaseHandler):
             self.write({'success': False, 'error': 'Failed returning results'})
 
 
-def addView(route, func, static = False):
+def addView(route, func):
     views[route] = func
 
 

couchpotato/api.py

@@ -3,13 +3,11 @@ from threading import Thread
 import json
 import threading
 import traceback
-import urllib
+from six.moves import urllib
 
 from couchpotato.core.helpers.request import getParams
 from couchpotato.core.logger import CPLog
-from tornado.gen import coroutine
 from tornado.web import RequestHandler, asynchronous
-import tornado
 
 
 log = CPLog(__name__)
@@ -28,10 +26,18 @@ def run_async(func):
     def async_func(*args, **kwargs):
         func_hl = Thread(target = func, args = args, kwargs = kwargs)
         func_hl.start()
-        return func_hl
 
     return async_func
 
+@run_async
+def run_handler(route, kwargs, callback = None):
+    try:
+        res = api[route](**kwargs)
+        callback(res, route)
+    except:
+        log.error('Failed doing api request "%s": %s', (route, traceback.format_exc()))
+        callback({'success': False, 'error': 'Failed returning results'}, route)
+
 
 # NonBlock API handler
 class NonBlockHandler(RequestHandler):
@@ -78,20 +84,25 @@ def addNonBlockApiView(route, func_tuple, docs = None, **kwargs):
 # Blocking API handler
 class ApiHandler(RequestHandler):
 
-    @coroutine
+    @asynchronous
     def get(self, route, *args, **kwargs):
         route = route.strip('/')
         if not api.get(route):
             self.write('API call doesn\'t seem to exist')
+            self.finish()
             return
 
+        # Create lock if it doesn't exist
+        if route in api_locks and not api_locks.get(route):
+            api_locks[route] = threading.Lock()
+
         api_locks[route].acquire()
 
         try:
 
             kwargs = {}
             for x in self.request.arguments:
-                kwargs[x] = urllib.unquote(self.get_argument(x))
+                kwargs[x] = urllib.parse.unquote(self.get_argument(x))
 
             # Split array arguments
             kwargs = getParams(kwargs)
@@ -102,36 +113,45 @@ class ApiHandler(RequestHandler):
             except: pass
 
             # Add async callback handler
-            @run_async
-            def run_handler(callback):
-                try:
-                    res = api[route](**kwargs)
-                    callback(res)
-                except:
-                    log.error('Failed doing api request "%s": %s', (route, traceback.format_exc()))
-                    callback({'success': False, 'error': 'Failed returning results'})
-
-            result = yield tornado.gen.Task(run_handler)
-
-            # Check JSONP callback
-            jsonp_callback = self.get_argument('callback_func', default = None)
-
-            if jsonp_callback:
-                self.write(str(jsonp_callback) + '(' + json.dumps(result) + ')')
-                self.set_header("Content-Type", "text/javascript")
-            elif isinstance(result, tuple) and result[0] == 'redirect':
-                self.redirect(result[1])
-            else:
-                self.write(result)
+            run_handler(route, kwargs, callback = self.taskFinished)
 
         except:
             log.error('Failed doing api request "%s": %s', (route, traceback.format_exc()))
-            self.write({'success': False, 'error': 'Failed returning results'})
+            try:
+                self.write({'success': False, 'error': 'Failed returning results'})
+                self.finish()
+            except:
+                log.error('Failed write error "%s": %s', (route, traceback.format_exc()))
 
-        api_locks[route].release()
+            api_locks[route].release()
 
     post = get
 
+    def taskFinished(self, result, route):
+
+        if not self.request.connection.stream.closed():
+            try:
+                # Check JSONP callback
+                jsonp_callback = self.get_argument('callback_func', default = None)
+
+                if jsonp_callback:
+                    self.write(str(jsonp_callback) + '(' + json.dumps(result) + ')')
+                    self.set_header("Content-Type", "text/javascript")
+                    self.finish()
+                elif isinstance(result, tuple) and result[0] == 'redirect':
+                    self.redirect(result[1])
+                else:
+                    self.write(result)
+                    self.finish()
+            except UnicodeDecodeError:
+                log.error('Failed proper encode: %s', traceback.format_exc())
+            except:
+                log.debug('Failed doing request, probably already closed: %s', (traceback.format_exc()))
+                try: self.finish({'success': False, 'error': 'Failed returning results'})
+                except: pass
+
+        api_locks[route].release()
+
 
 def addApiView(route, func, static = False, docs = None, **kwargs):
 

couchpotato/core/_base/_core.py

@@ -8,7 +8,7 @@ import webbrowser
 
 from couchpotato.api import addApiView
 from couchpotato.core.event import fireEvent, addEvent
-from couchpotato.core.helpers.variable import cleanHost, md5
+from couchpotato.core.helpers.variable import cleanHost, md5, isSubFolder
 from couchpotato.core.logger import CPLog
 from couchpotato.core.plugins.base import Plugin
 from couchpotato.environment import Env
@@ -71,13 +71,14 @@ class Core(Plugin):
         return value if value and len(value) > 3 else uuid4().hex
 
     def checkDataDir(self):
-        if Env.get('app_dir') in Env.get('data_dir'):
+        if isSubFolder(Env.get('data_dir'), Env.get('app_dir')):
             log.error('You should NOT use your CouchPotato directory to save your settings in. Files will get overwritten or be deleted.')
 
         return True
 
     def cleanUpFolders(self):
-        self.deleteEmptyFolder(Env.get('app_dir'), show_error = False)
+        only_clean = ['couchpotato', 'libs', 'init']
+        self.deleteEmptyFolder(Env.get('app_dir'), show_error = False, only_clean = only_clean)
 
     def available(self, **kwargs):
         return {
@@ -90,7 +91,11 @@ class Core(Plugin):
 
         def shutdown():
             self.initShutdown()
-        IOLoop.current().add_callback(shutdown)
+
+        if IOLoop.current()._closing:
+            shutdown()
+        else:
+            IOLoop.current().add_callback(shutdown)
 
         return 'shutdown'
 
@@ -113,7 +118,7 @@ class Core(Plugin):
 
         self.shutdown_started = True
 
-        fireEvent('app.do_shutdown')
+        fireEvent('app.do_shutdown', restart = restart)
         log.debug('Every plugin got shutdown event')
 
         loop = True
@@ -138,8 +143,11 @@ class Core(Plugin):
 
         log.debug('Safe to shutdown/restart')
 
+        loop = IOLoop.current()
+
         try:
-            IOLoop.current().stop()
+            if not loop._closing:
+                loop.stop()
         except RuntimeError:
             pass
         except:
@@ -278,13 +286,13 @@ config = [{
                     'name': 'permission_folder',
                     'default': '0755',
                     'label': 'Folder CHMOD',
-                    'description': 'Can be either decimal (493) or octal (leading zero: 0755)',
+                    'description': 'Can be either decimal (493) or octal (leading zero: 0755). <a target="_blank" href="http://permissions-calculator.org/">Calculate the correct value</a>',
                 },
                 {
                     'name': 'permission_file',
                     'default': '0755',
                     'label': 'File CHMOD',
-                    'description': 'Same as Folder CHMOD but for files',
+                    'description': 'See Folder CHMOD description, but for files',
                 },
             ],
         },

couchpotato/core/_base/downloader/main.py

@@ -25,6 +25,7 @@ class DownloaderBase(Provider):
     status_support = True
 
     torrent_sources = [
+        'https://zoink.it/torrent/%s.torrent',
         'http://torrage.com/torrent/%s.torrent',
         'https://torcache.net/torrent/%s.torrent',
     ]
@@ -72,6 +73,9 @@ class DownloaderBase(Provider):
             return
         return self.download(data = data, media = media, filedata = filedata)
 
+    def download(self, *args, **kwargs):
+        return False
+
     def _getAllDownloadStatus(self, download_ids):
         if self.isDisabled(manual = True, data = {}):
             return

couchpotato/core/_base/downloader/static/downloaders.js

@@ -40,15 +40,16 @@ var DownloadersBase = new Class({
 
 								button.set('text', button_name);
 
+								var message;
 								if(json.success){
-									var message = new Element('span.success', {
+									message = new Element('span.success', {
 										'text': 'Connection successful'
 									}).inject(button, 'after')
 								}
 								else {
 									var msg_text = 'Connection failed. Check logs for details.';
 									if(json.hasOwnProperty('msg')) msg_text = json.msg;
-									var message = new Element('span.failed', {
+									message = new Element('span.failed', {
 										'text': msg_text
 									}).inject(button, 'after')
 								}

couchpotato/core/_base/scheduler.py

@@ -33,9 +33,9 @@ class Scheduler(Plugin):
             except:
                 pass
 
-    def doShutdown(self):
+    def doShutdown(self, *args, **kwargs):
         self.stop()
-        return super(Scheduler, self).doShutdown()
+        return super(Scheduler, self).doShutdown(*args, **kwargs)
 
     def stop(self):
         if self.started:

couchpotato/core/_base/updater/main.py

@@ -10,13 +10,13 @@ from threading import RLock
 
 from couchpotato.api import addApiView
 from couchpotato.core.event import addEvent, fireEvent, fireEventAsync
-from couchpotato.core.helpers.encoding import ss
+from couchpotato.core.helpers.encoding import sp
+from couchpotato.core.helpers.variable import removePyc
 from couchpotato.core.logger import CPLog
 from couchpotato.core.plugins.base import Plugin
 from couchpotato.environment import Env
 from dateutil.parser import parse
 from git.repository import LocalRepository
-from scandir import scandir
 import version
 from six.moves import filter
 
@@ -142,9 +142,11 @@ class Updater(Plugin):
             'success': success
         }
 
-    def doShutdown(self):
-        self.updater.deletePyc(show_logs = False)
-        return super(Updater, self).doShutdown()
+    def doShutdown(self, *args, **kwargs):
+        if not Env.get('dev') and not Env.get('desktop'):
+            removePyc(Env.get('app_dir'), show_logs = False)
+
+        return super(Updater, self).doShutdown(*args, **kwargs)
 
 
 class BaseUpdater(Plugin):
@@ -180,30 +182,6 @@ class BaseUpdater(Plugin):
     def check(self):
         pass
 
-    def deletePyc(self, only_excess = True, show_logs = True):
-
-        for root, dirs, files in scandir.walk(ss(Env.get('app_dir'))):
-
-            pyc_files = filter(lambda filename: filename.endswith('.pyc'), files)
-            py_files = set(filter(lambda filename: filename.endswith('.py'), files))
-            excess_pyc_files = filter(lambda pyc_filename: pyc_filename[:-1] not in py_files, pyc_files) if only_excess else pyc_files
-
-            for excess_pyc_file in excess_pyc_files:
-                full_path = os.path.join(root, excess_pyc_file)
-                if show_logs: log.debug('Removing old PYC file: %s', full_path)
-                try:
-                    os.remove(full_path)
-                except:
-                    log.error('Couldn\'t remove %s: %s', (full_path, traceback.format_exc()))
-
-            for dir_name in dirs:
-                full_path = os.path.join(root, dir_name)
-                if len(os.listdir(full_path)) == 0:
-                    try:
-                        os.rmdir(full_path)
-                    except:
-                        log.error('Couldn\'t remove empty directory %s: %s', (full_path, traceback.format_exc()))
-
 
 class GitUpdater(BaseUpdater):
 
@@ -322,17 +300,18 @@ class SourceUpdater(BaseUpdater):
         return False
 
     def replaceWith(self, path):
-        app_dir = ss(Env.get('app_dir'))
-        data_dir = ss(Env.get('data_dir'))
+        path = sp(path)
+        app_dir = Env.get('app_dir')
+        data_dir = Env.get('data_dir')
 
         # Get list of files we want to overwrite
-        self.deletePyc()
+        removePyc(app_dir)
         existing_files = []
-        for root, subfiles, filenames in scandir.walk(app_dir):
+        for root, subfiles, filenames in os.walk(app_dir):
             for filename in filenames:
                 existing_files.append(os.path.join(root, filename))
 
-        for root, subfiles, filenames in scandir.walk(path):
+        for root, subfiles, filenames in os.walk(path):
             for filename in filenames:
                 fromfile = os.path.join(root, filename)
                 tofile = os.path.join(app_dir, fromfile.replace(path + os.path.sep, ''))

couchpotato/core/database.py

@@ -2,12 +2,15 @@ import json
 import os
 import time
 import traceback
+from sqlite3 import OperationalError
+from CodernityDB3.index import Index
 
 from couchpotato import CPLog
 from couchpotato.api import addApiView
-from couchpotato.core.event import addEvent, fireEvent
-from couchpotato.core.helpers.encoding import toUnicode
-from couchpotato.core.helpers.variable import getImdb, tryInt
+from couchpotato.core.event import addEvent, fireEvent, fireEventAsync
+from couchpotato.core.helpers.database import IndexException, IndexNotFoundException, IndexConflict, RecordNotFound
+from couchpotato.core.helpers.encoding import toUnicode, sp
+from couchpotato.core.helpers.variable import getImdb, tryInt, randomString
 
 
 log = CPLog(__name__)
@@ -15,19 +18,25 @@ log = CPLog(__name__)
 
 class Database(object):
 
-    indexes = []
+    indexes = None
     db = None
 
     def __init__(self):
 
+        self.indexes = {}
+
         addApiView('database.list_documents', self.listDocuments)
         addApiView('database.reindex', self.reindex)
         addApiView('database.compact', self.compact)
         addApiView('database.document.update', self.updateDocument)
         addApiView('database.document.delete', self.deleteDocument)
 
+        addEvent('database.setup.after', self.startup_compact)
         addEvent('database.setup_index', self.setupIndex)
+        addEvent('database.delete_corrupted', self.deleteCorrupted)
+
         addEvent('app.migrate', self.migrate)
+        addEvent('app.after_shutdown', self.close)
 
     def getDB(self):
 
@@ -37,28 +46,50 @@ class Database(object):
 
         return self.db
 
+    def close(self, **kwargs):
+        self.getDB().close()
+
     def setupIndex(self, index_name, klass):
 
-        self.indexes.append(index_name)
+        self.indexes[index_name] = klass
 
         db = self.getDB()
 
         # Category index
         index_instance = klass(db.path, index_name)
         try:
-            db.add_index(index_instance)
-            db.reindex_index(index_name)
-        except:
-            previous = db.indexes_names[index_name]
-            previous_version = previous._version
-            current_version = klass._version
 
-            # Only edit index if versions are different
-            if previous_version < current_version:
-                log.debug('Index "%s" already exists, updating and reindexing', index_name)
-                db.destroy_index(previous)
+            # Make sure store and bucket don't exist
+            exists = []
+            for x in ['buck', 'stor']:
+                full_path = os.path.join(db.path, '%s_%s' % (index_name, x))
+                if os.path.exists(full_path):
+                    exists.append(full_path)
+
+            if index_name not in db.indexes_names:
+
+                # Remove existing buckets if index isn't there
+                for x in exists:
+                    os.unlink(x)
+
+                # Add index (will restore buckets)
                 db.add_index(index_instance)
                 db.reindex_index(index_name)
+            else:
+                # Previous info
+                previous = db.indexes_names[index_name]
+                previous_version = previous._version
+                current_version = klass._version
+
+                # Only edit index if versions are different
+                if previous_version < current_version:
+                    log.debug('Index "%s" already exists, updating and reindexing', index_name)
+                    db.destroy_index(previous)
+                    db.add_index(index_instance)
+                    db.reindex_index(index_name)
+
+        except:
+            log.error('Failed adding index %s: %s', (index_name, traceback.format_exc()))
 
     def deleteDocument(self, **kwargs):
 
@@ -118,6 +149,17 @@ class Database(object):
 
         return results
 
+    def deleteCorrupted(self, _id, traceback_error = ''):
+
+        db = self.getDB()
+
+        try:
+            log.debug('Deleted corrupted document "%s": %s', (_id, traceback_error))
+            corrupted = db.get('id', _id, with_storage = False)
+            db._delete_id_index(corrupted.get('_id'), corrupted.get('_rev'), None)
+        except:
+            log.debug('Failed deleting corrupted: %s', traceback.format_exc())
+
     def reindex(self, **kwargs):
 
         success = True
@@ -132,20 +174,108 @@ class Database(object):
             'success': success
         }
 
-    def compact(self, **kwargs):
+    def compact(self, try_repair = True, **kwargs):
+
+        success = False
+        db = self.getDB()
+
+        # Removing left over compact files
+        db_path = sp(db.path)
+        for f in os.listdir(sp(db.path)):
+            for x in ['_compact_buck', '_compact_stor']:
+                if f[-len(x):] == x:
+                    os.unlink(os.path.join(db_path, f))
 
-        success = True
         try:
-            db = self.getDB()
+            start = time.time()
+            size = float(db.get_db_details().get('size', 0))
+            log.debug('Compacting database, current size: %sMB', round(size/1048576, 2))
+
             db.compact()
+            new_size = float(db.get_db_details().get('size', 0))
+            log.debug('Done compacting database in %ss, new size: %sMB, saved: %sMB', (round(time.time()-start, 2), round(new_size/1048576, 2), round((size-new_size)/1048576, 2)))
+            success = True
+        except (IndexException, AttributeError):
+            if try_repair:
+                log.error('Something wrong with indexes, trying repair')
+
+                # Remove all indexes
+                old_indexes = self.indexes.keys()
+                for index_name in old_indexes:
+                    try:
+                        db.destroy_index(index_name)
+                    except IndexNotFoundException:
+                        pass
+                    except:
+                        log.error('Failed removing old index %s', index_name)
+
+                # Add them again
+                for index_name in self.indexes:
+                    klass = self.indexes[index_name]
+
+                    # Category index
+                    index_instance = klass(db.path, index_name)
+                    try:
+                        db.add_index(index_instance)
+                        db.reindex_index(index_name)
+                    except IndexConflict:
+                        pass
+                    except:
+                        log.error('Failed adding index %s', index_name)
+                        raise
+
+                self.compact(try_repair = False)
+            else:
+                log.error('Failed compact: %s', traceback.format_exc())
+
         except:
             log.error('Failed compact: %s', traceback.format_exc())
-            success = False
 
         return {
             'success': success
         }
 
+    # Compact on start
+    def startup_compact(self):
+        from couchpotato import Env
+
+        db = self.getDB()
+
+        # Try fix for migration failures on desktop
+        if Env.get('desktop'):
+            try:
+                list(db.all('profile', with_doc = True))
+            except RecordNotFound:
+
+                failed_location = '%s_failed' % db.path
+                old_db = os.path.join(Env.get('data_dir'), 'couchpotato.db.old')
+
+                if not os.path.isdir(failed_location) and os.path.isfile(old_db):
+                    log.error('Corrupt database, trying migrate again')
+                    db.close()
+
+                    # Rename database folder
+                    os.rename(db.path, '%s_failed' % db.path)
+
+                    # Rename .old database to try another migrate
+                    os.rename(old_db, old_db[:-4])
+
+                    fireEventAsync('app.restart')
+                else:
+                    log.error('Migration failed and couldn\'t recover database. Please report on GitHub, with this message.')
+                    db.reindex()
+
+                return
+
+        # Check size and compact if needed
+        size = db.get_db_details().get('size')
+        prop_name = 'last_db_compact'
+        last_check = int(Env.prop(prop_name, default = 0))
+
+        if size > 26214400 and last_check < time.time()-604800: # 25MB / 7 days
+            self.compact()
+            Env.prop(prop_name, value = int(time.time()))
+
     def migrate(self):
 
         from couchpotato import Env
@@ -182,301 +312,326 @@ class Database(object):
             }
 
             migrate_data = {}
+            rename_old = False
 
-            c = conn.cursor()
+            try:
 
-            for ml in migrate_list:
-                migrate_data[ml] = {}
-                rows = migrate_list[ml]
+                c = conn.cursor()
 
-                try:
-                    c.execute('SELECT %s FROM `%s`' % ('`' + '`,`'.join(rows) + '`', ml))
-                except:
-                    # ignore faulty destination_id database
-                    if ml == 'category':
-                        migrate_data[ml] = {}
+                for ml in migrate_list:
+                    migrate_data[ml] = {}
+                    rows = migrate_list[ml]
+
+                    try:
+                        c.execute('SELECT %s FROM `%s`' % ('`' + '`,`'.join(rows) + '`', ml))
+                    except:
+                        # ignore faulty destination_id database
+                        if ml == 'category':
+                            migrate_data[ml] = {}
+                        else:
+                            rename_old = True
+                            raise
+
+                    for p in c.fetchall():
+                        columns = {}
+                        for row in migrate_list[ml]:
+                            columns[row] = p[rows.index(row)]
+
+                        if not migrate_data[ml].get(p[0]):
+                            migrate_data[ml][p[0]] = columns
+                        else:
+                            if not isinstance(migrate_data[ml][p[0]], list):
+                                migrate_data[ml][p[0]] = [migrate_data[ml][p[0]]]
+                            migrate_data[ml][p[0]].append(columns)
+
+                conn.close()
+
+                log.info('Getting data took %s', time.time() - migrate_start)
+
+                db = self.getDB()
+                if not db.opened:
+                    return
+
+                # Use properties
+                properties = migrate_data['properties']
+                log.info('Importing %s properties', len(properties))
+                for x in properties:
+                    property = properties[x]
+                    Env.prop(property.get('identifier'), property.get('value'))
+
+                # Categories
+                categories = migrate_data.get('category', [])
+                log.info('Importing %s categories', len(categories))
+                category_link = {}
+                for x in categories:
+                    c = categories[x]
+
+                    new_c = db.insert({
+                        '_t': 'category',
+                        'order': c.get('order', 999),
+                        'label': toUnicode(c.get('label', '')),
+                        'ignored': toUnicode(c.get('ignored', '')),
+                        'preferred': toUnicode(c.get('preferred', '')),
+                        'required': toUnicode(c.get('required', '')),
+                        'destination': toUnicode(c.get('destination', '')),
+                    })
+
+                    category_link[x] = new_c.get('_id')
+
+                # Profiles
+                log.info('Importing profiles')
+                new_profiles = db.all('profile', with_doc = True)
+                new_profiles_by_label = {}
+                for x in new_profiles:
+
+                    # Remove default non core profiles
+                    if not x['doc'].get('core'):
+                        db.delete(x['doc'])
                     else:
-                        raise
+                        new_profiles_by_label[x['doc']['label']] = x['_id']
 
-                for p in c.fetchall():
-                    columns = {}
-                    for row in migrate_list[ml]:
-                        columns[row] = p[rows.index(row)]
+                profiles = migrate_data['profile']
+                profile_link = {}
+                for x in profiles:
+                    p = profiles[x]
 
-                    if not migrate_data[ml].get(p[0]):
-                        migrate_data[ml][p[0]] = columns
+                    exists = new_profiles_by_label.get(p.get('label'))
+
+                    # Update existing with order only
+                    if exists and p.get('core'):
+                        profile = db.get('id', exists)
+                        profile['order'] = tryInt(p.get('order'))
+                        profile['hide'] = p.get('hide') in [1, True, 'true', 'True']
+                        db.update(profile)
+
+                        profile_link[x] = profile.get('_id')
                     else:
-                        if not isinstance(migrate_data[ml][p[0]], list):
-                            migrate_data[ml][p[0]] = [migrate_data[ml][p[0]]]
-                        migrate_data[ml][p[0]].append(columns)
 
-            conn.close()
-
-            log.info('Getting data took %s', time.time() - migrate_start)
-
-            db = self.getDB()
-
-            # Use properties
-            properties = migrate_data['properties']
-            log.info('Importing %s properties', len(properties))
-            for x in properties:
-                property = properties[x]
-                Env.prop(property.get('identifier'), property.get('value'))
-
-            # Categories
-            categories = migrate_data.get('category', [])
-            log.info('Importing %s categories', len(categories))
-            category_link = {}
-            for x in categories:
-                c = categories[x]
-
-                new_c = db.insert({
-                    '_t': 'category',
-                    'order': c.get('order', 999),
-                    'label': toUnicode(c.get('label', '')),
-                    'ignored': toUnicode(c.get('ignored', '')),
-                    'preferred': toUnicode(c.get('preferred', '')),
-                    'required': toUnicode(c.get('required', '')),
-                    'destination': toUnicode(c.get('destination', '')),
-                })
-
-                category_link[x] = new_c.get('_id')
-
-            # Profiles
-            log.info('Importing profiles')
-            new_profiles = db.all('profile', with_doc = True)
-            new_profiles_by_label = {}
-            for x in new_profiles:
-
-                # Remove default non core profiles
-                if not x['doc'].get('core'):
-                    db.delete(x['doc'])
-                else:
-                    new_profiles_by_label[x['doc']['label']] = x['_id']
-
-            profiles = migrate_data['profile']
-            profile_link = {}
-            for x in profiles:
-                p = profiles[x]
-
-                exists = new_profiles_by_label.get(p.get('label'))
-
-                # Update existing with order only
-                if exists and p.get('core'):
-                    profile = db.get('id', exists)
-                    profile['order'] = tryInt(p.get('order'))
-                    profile['hide'] = p.get('hide') in [1, True, 'true', 'True']
-                    db.update(profile)
-
-                    profile_link[x] = profile.get('_id')
-                else:
-
-                    new_profile = {
-                        '_t': 'profile',
-                        'label': p.get('label'),
-                        'order': int(p.get('order', 999)),
-                        'core': p.get('core', False),
-                        'qualities': [],
-                        'wait_for': [],
-                        'finish': []
-                    }
-
-                    types = migrate_data['profiletype']
-                    for profile_type in types:
-                        p_type = types[profile_type]
-                        if types[profile_type]['profile_id'] == p['id']:
-                            new_profile['finish'].append(p_type['finish'])
-                            new_profile['wait_for'].append(p_type['wait_for'])
-                            new_profile['qualities'].append(migrate_data['quality'][p_type['quality_id']]['identifier'])
-
-                    new_profile.update(db.insert(new_profile))
-
-                    profile_link[x] = new_profile.get('_id')
-
-            # Qualities
-            log.info('Importing quality sizes')
-            new_qualities = db.all('quality', with_doc = True)
-            new_qualities_by_identifier = {}
-            for x in new_qualities:
-                new_qualities_by_identifier[x['doc']['identifier']] = x['_id']
-
-            qualities = migrate_data['quality']
-            quality_link = {}
-            for x in qualities:
-                q = qualities[x]
-                q_id = new_qualities_by_identifier[q.get('identifier')]
-
-                quality = db.get('id', q_id)
-                quality['order'] = q.get('order')
-                quality['size_min'] = tryInt(q.get('size_min'))
-                quality['size_max'] = tryInt(q.get('size_max'))
-                db.update(quality)
-
-                quality_link[x] = quality
-
-            # Titles
-            titles = migrate_data['librarytitle']
-            titles_by_library = {}
-            for x in titles:
-                title = titles[x]
-                if title.get('default'):
-                    titles_by_library[title.get('libraries_id')] = title.get('title')
-
-            # Releases
-            releaseinfos = migrate_data['releaseinfo']
-            for x in releaseinfos:
-                info = releaseinfos[x]
-
-                # Skip if release doesn't exist for this info
-                if not migrate_data['release'].get(info.get('release_id')):
-                    continue
-
-                if not migrate_data['release'][info.get('release_id')].get('info'):
-                    migrate_data['release'][info.get('release_id')]['info'] = {}
-
-                migrate_data['release'][info.get('release_id')]['info'][info.get('identifier')] = info.get('value')
-
-            releases = migrate_data['release']
-            releases_by_media = {}
-            for x in releases:
-                release = releases[x]
-                if not releases_by_media.get(release.get('movie_id')):
-                    releases_by_media[release.get('movie_id')] = []
-
-                releases_by_media[release.get('movie_id')].append(release)
-
-            # Type ids
-            types = migrate_data['filetype']
-            type_by_id = {}
-            for t in types:
-                type = types[t]
-                type_by_id[type.get('id')] = type
-
-            # Media
-            log.info('Importing %s media items', len(migrate_data['movie']))
-            statuses = migrate_data['status']
-            libraries = migrate_data['library']
-            library_files = migrate_data['library_files__file_library']
-            releases_files = migrate_data['release_files__file_release']
-            all_files = migrate_data['file']
-            poster_type = migrate_data['filetype']['poster']
-            medias = migrate_data['movie']
-            for x in medias:
-                m = medias[x]
-
-                status = statuses.get(m['status_id']).get('identifier')
-                l = libraries[m['library_id']]
-
-                # Only migrate wanted movies, Skip if no identifier present
-                if not getImdb(l.get('identifier')): continue
-
-                profile_id = profile_link.get(m['profile_id'])
-                category_id = category_link.get(m['category_id'])
-                title = titles_by_library.get(m['library_id'])
-                releases = releases_by_media.get(x, [])
-                info = json.loads(l.get('info', ''))
-
-                files = library_files.get(m['library_id'], [])
-                if not isinstance(files, list):
-                    files = [files]
-
-                added_media = fireEvent('movie.add', {
-                    'info': info,
-                    'identifier': l.get('identifier'),
-                    'profile_id': profile_id,
-                    'category_id': category_id,
-                    'title': title
-                }, force_readd = False, search_after = False, update_after = False, notify_after = False, status = status, single = True)
-
-                if not added_media:
-                    log.error('Failed adding media %s: %s', (l.get('identifier'), info))
-                    continue
-
-                added_media['files'] = added_media.get('files', {})
-                for f in files:
-                    ffile = all_files[f.get('file_id')]
-
-                    # Only migrate posters
-                    if ffile.get('type_id') == poster_type.get('id'):
-                        if ffile.get('path') not in added_media['files'].get('image_poster', []) and os.path.isfile(ffile.get('path')):
-                            added_media['files']['image_poster'] = [ffile.get('path')]
-                            break
-
-                if 'image_poster' in added_media['files']:
-                    db.update(added_media)
-
-                for rel in releases:
-
-                    empty_info = False
-                    if not rel.get('info'):
-                        empty_info = True
-                        rel['info'] = {}
-
-                    quality = quality_link[rel.get('quality_id')]
-                    release_status = statuses.get(rel.get('status_id')).get('identifier')
-
-                    if rel['info'].get('download_id'):
-                        status_support = rel['info'].get('download_status_support', False) in [True, 'true', 'True']
-                        rel['info']['download_info'] = {
-                            'id': rel['info'].get('download_id'),
-                            'downloader': rel['info'].get('download_downloader'),
-                            'status_support': status_support,
+                        new_profile = {
+                            '_t': 'profile',
+                            'label': p.get('label'),
+                            'order': int(p.get('order', 999)),
+                            'core': p.get('core', False),
+                            'qualities': [],
+                            'wait_for': [],
+                            'finish': []
                         }
 
-                    # Add status to keys
-                    rel['info']['status'] = release_status
-                    if not empty_info:
-                        fireEvent('release.create_from_search', [rel['info']], added_media, quality, single = True)
-                    else:
-                        release = {
-                            '_t': 'release',
-                            'identifier': rel.get('identifier'),
-                            'media_id': added_media.get('_id'),
-                            'quality': quality.get('identifier'),
-                            'status': release_status,
-                            'last_edit': int(time.time()),
-                            'files': {}
-                        }
+                        types = migrate_data['profiletype']
+                        for profile_type in types:
+                            p_type = types[profile_type]
+                            if types[profile_type]['profile_id'] == p['id']:
+                                if p_type['quality_id']:
+                                    new_profile['finish'].append(p_type['finish'])
+                                    new_profile['wait_for'].append(p_type['wait_for'])
+                                    new_profile['qualities'].append(migrate_data['quality'][p_type['quality_id']]['identifier'])
 
-                        # Add downloader info if provided
-                        try:
-                            release['download_info'] = rel['info']['download_info']
-                            del rel['download_info']
-                        except:
-                            pass
+                        if len(new_profile['qualities']) > 0:
+                            new_profile.update(db.insert(new_profile))
+                            profile_link[x] = new_profile.get('_id')
+                        else:
+                            log.error('Corrupt profile list for "%s", using default.', p.get('label'))
 
-                        # Add files
-                        release_files = releases_files.get(rel.get('id'), [])
-                        if not isinstance(release_files, list):
-                            release_files = [release_files]
+                # Qualities
+                log.info('Importing quality sizes')
+                new_qualities = db.all('quality', with_doc = True)
+                new_qualities_by_identifier = {}
+                for x in new_qualities:
+                    new_qualities_by_identifier[x['doc']['identifier']] = x['_id']
 
-                        if len(release_files) == 0:
+                qualities = migrate_data['quality']
+                quality_link = {}
+                for x in qualities:
+                    q = qualities[x]
+                    q_id = new_qualities_by_identifier[q.get('identifier')]
+
+                    quality = db.get('id', q_id)
+                    quality['order'] = q.get('order')
+                    quality['size_min'] = tryInt(q.get('size_min'))
+                    quality['size_max'] = tryInt(q.get('size_max'))
+                    db.update(quality)
+
+                    quality_link[x] = quality
+
+                # Titles
+                titles = migrate_data['librarytitle']
+                titles_by_library = {}
+                for x in titles:
+                    title = titles[x]
+                    if title.get('default'):
+                        titles_by_library[title.get('libraries_id')] = title.get('title')
+
+                # Releases
+                releaseinfos = migrate_data['releaseinfo']
+                for x in releaseinfos:
+                    info = releaseinfos[x]
+
+                    # Skip if release doesn't exist for this info
+                    if not migrate_data['release'].get(info.get('release_id')):
+                        continue
+
+                    if not migrate_data['release'][info.get('release_id')].get('info'):
+                        migrate_data['release'][info.get('release_id')]['info'] = {}
+
+                    migrate_data['release'][info.get('release_id')]['info'][info.get('identifier')] = info.get('value')
+
+                releases = migrate_data['release']
+                releases_by_media = {}
+                for x in releases:
+                    release = releases[x]
+                    if not releases_by_media.get(release.get('movie_id')):
+                        releases_by_media[release.get('movie_id')] = []
+
+                    releases_by_media[release.get('movie_id')].append(release)
+
+                # Type ids
+                types = migrate_data['filetype']
+                type_by_id = {}
+                for t in types:
+                    type = types[t]
+                    type_by_id[type.get('id')] = type
+
+                # Media
+                log.info('Importing %s media items', len(migrate_data['movie']))
+                statuses = migrate_data['status']
+                libraries = migrate_data['library']
+                library_files = migrate_data['library_files__file_library']
+                releases_files = migrate_data['release_files__file_release']
+                all_files = migrate_data['file']
+                poster_type = migrate_data['filetype']['poster']
+                medias = migrate_data['movie']
+                for x in medias:
+                    m = medias[x]
+
+                    status = statuses.get(m['status_id']).get('identifier')
+                    l = libraries.get(m['library_id'])
+
+                    # Only migrate wanted movies, Skip if no identifier present
+                    if not l or not getImdb(l.get('identifier')): continue
+
+                    profile_id = profile_link.get(m['profile_id'])
+                    category_id = category_link.get(m['category_id'])
+                    title = titles_by_library.get(m['library_id'])
+                    releases = releases_by_media.get(x, [])
+                    info = json.loads(l.get('info', ''))
+
+                    files = library_files.get(m['library_id'], [])
+                    if not isinstance(files, list):
+                        files = [files]
+
+                    added_media = fireEvent('movie.add', {
+                        'info': info,
+                        'identifier': l.get('identifier'),
+                        'profile_id': profile_id,
+                        'category_id': category_id,
+                        'title': title
+                    }, force_readd = False, search_after = False, update_after = False, notify_after = False, status = status, single = True)
+
+                    if not added_media:
+                        log.error('Failed adding media %s: %s', (l.get('identifier'), info))
+                        continue
+
+                    added_media['files'] = added_media.get('files', {})
+                    for f in files:
+                        ffile = all_files[f.get('file_id')]
+
+                        # Only migrate posters
+                        if ffile.get('type_id') == poster_type.get('id'):
+                            if ffile.get('path') not in added_media['files'].get('image_poster', []) and os.path.isfile(ffile.get('path')):
+                                added_media['files']['image_poster'] = [ffile.get('path')]
+                                break
+
+                    if 'image_poster' in added_media['files']:
+                        db.update(added_media)
+
+                    for rel in releases:
+
+                        empty_info = False
+                        if not rel.get('info'):
+                            empty_info = True
+                            rel['info'] = {}
+
+                        quality = quality_link.get(rel.get('quality_id'))
+                        if not quality:
                             continue
 
-                        for f in release_files:
-                            rfile = all_files[f.get('file_id')]
-                            file_type = type_by_id.get(rfile.get('type_id')).get('identifier')
+                        release_status = statuses.get(rel.get('status_id')).get('identifier')
 
-                            if not release['files'].get(file_type):
-                                release['files'][file_type] = []
+                        if rel['info'].get('download_id'):
+                            status_support = rel['info'].get('download_status_support', False) in [True, 'true', 'True']
+                            rel['info']['download_info'] = {
+                                'id': rel['info'].get('download_id'),
+                                'downloader': rel['info'].get('download_downloader'),
+                                'status_support': status_support,
+                            }
 
-                            release['files'][file_type].append(rfile.get('path'))
+                        # Add status to keys
+                        rel['info']['status'] = release_status
+                        if not empty_info:
+                            fireEvent('release.create_from_search', [rel['info']], added_media, quality, single = True)
+                        else:
+                            release = {
+                                '_t': 'release',
+                                'identifier': rel.get('identifier'),
+                                'media_id': added_media.get('_id'),
+                                'quality': quality.get('identifier'),
+                                'status': release_status,
+                                'last_edit': int(time.time()),
+                                'files': {}
+                            }
 
-                        try:
-                            rls = db.get('release_identifier', rel.get('identifier'), with_doc = True)['doc']
-                            rls.update(release)
-                            db.update(rls)
-                        except:
-                            db.insert(release)
+                            # Add downloader info if provided
+                            try:
+                                release['download_info'] = rel['info']['download_info']
+                                del rel['download_info']
+                            except:
+                                pass
+
+                            # Add files
+                            release_files = releases_files.get(rel.get('id'), [])
+                            if not isinstance(release_files, list):
+                                release_files = [release_files]
+
+                            if len(release_files) == 0:
+                                continue
+
+                            for f in release_files:
+                                rfile = all_files.get(f.get('file_id'))
+                                if not rfile:
+                                    continue
+
+                                file_type = type_by_id.get(rfile.get('type_id')).get('identifier')
+
+                                if not release['files'].get(file_type):
+                                    release['files'][file_type] = []
+
+                                release['files'][file_type].append(rfile.get('path'))
+
+                            try:
+                                rls = db.get('release_identifier', rel.get('identifier'), with_doc = True)['doc']
+                                rls.update(release)
+                                db.update(rls)
+                            except:
+                                db.insert(release)
+
+                log.info('Total migration took %s', time.time() - migrate_start)
+                log.info('=' * 30)
+
+                rename_old = True
+
+            except OperationalError:
+                log.error('Migrating from faulty database, probably a (too) old version: %s', traceback.format_exc())
+            except:
+                log.error('Migration failed: %s', traceback.format_exc())
 
-            log.info('Total migration took %s', time.time() - migrate_start)
-            log.info('=' * 30)
 
             # rename old database
-            log.info('Renaming old database to %s ', old_db + '.old')
-            os.rename(old_db, old_db + '.old')
+            if rename_old:
+                random = randomString()
+                log.info('Renaming old database to %s ', '%s.%s_old' % (old_db, random))
+                os.rename(old_db, '%s.%s_old' % (old_db, random))
 
-            if os.path.isfile(old_db + '-wal'):
-                os.rename(old_db + '-wal', old_db + '-wal.old')
-            if os.path.isfile(old_db + '-shm'):
-                os.rename(old_db + '-shm', old_db + '-shm.old')
+                if os.path.isfile(old_db + '-wal'):
+                    os.rename(old_db + '-wal', '%s-wal.%s_old' % (old_db, random))
+                if os.path.isfile(old_db + '-shm'):
+                    os.rename(old_db + '-shm', '%s-shm.%s_old' % (old_db, random))

couchpotato/core/downloaders/nzbvortex.py

@@ -1,16 +1,10 @@
 from base64 import b64encode
-from urllib2 import URLError
+import os
 from uuid import uuid4
 import hashlib
-import httplib
-import json
-import os
-import socket
-import ssl
-import sys
-import time
 import traceback
-import urllib2
+
+from requests import HTTPError
 
 from couchpotato.core._base.downloader.main import DownloaderBase, ReleaseDownloadList
 from couchpotato.core.helpers.encoding import tryUrlencode, sp
@@ -35,13 +29,17 @@ class NZBVortex(DownloaderBase):
 
         # Send the nzb
         try:
-            nzb_filename = self.createFileName(data, filedata, media)
-            self.call('nzb/add', files = {'file': (nzb_filename, filedata)})
+            nzb_filename = self.createFileName(data, filedata, media, unique_tag = True)
+            response = self.call('nzb/add', files = {'file': (nzb_filename, filedata, 'application/octet-stream')}, parameters = {
+                'name': nzb_filename,
+                'groupname': self.conf('group')
+            })
 
-            time.sleep(10)
-            raw_statuses = self.call('nzb')
-            nzb_id = [nzb['id'] for nzb in raw_statuses.get('nzbs', []) if os.path.basename(nzb['nzbFileName']) == nzb_filename][0]
-            return self.downloadReturnId(nzb_id)
+            if response and response.get('result', '').lower() == 'ok':
+                return self.downloadReturnId(nzb_filename)
+
+            log.error('Something went wrong sending the NZB file. Response: %s', response)
+            return False
         except:
             log.error('Something went wrong sending the NZB file: %s', traceback.format_exc())
             return False
@@ -60,7 +58,8 @@ class NZBVortex(DownloaderBase):
 
         release_downloads = ReleaseDownloadList(self)
         for nzb in raw_statuses.get('nzbs', []):
-            if nzb['id'] in ids:
+            nzb_id = os.path.basename(nzb['nzbFileName'])
+            if nzb_id in ids:
 
                 # Check status
                 status = 'busy'
@@ -70,7 +69,8 @@ class NZBVortex(DownloaderBase):
                     status = 'failed'
 
                 release_downloads.append({
-                    'id': nzb['id'],
+                    'temp_id': nzb['id'],
+                    'id': nzb_id,
                     'name': nzb['uiTitle'],
                     'status': status,
                     'original_status': nzb['state'],
@@ -85,7 +85,7 @@ class NZBVortex(DownloaderBase):
         log.info('%s failed downloading, deleting...', release_download['name'])
 
         try:
-            self.call('nzb/%s/cancel' % release_download['id'])
+            self.call('nzb/%s/cancel' % release_download['temp_id'])
         except:
             log.error('Failed deleting: %s', traceback.format_exc(0))
             return False
@@ -114,7 +114,7 @@ class NZBVortex(DownloaderBase):
         log.error('Login failed, please check you api-key')
         return False
 
-    def call(self, call, parameters = None, repeat = False, auth = True, *args, **kwargs):
+    def call(self, call, parameters = None, is_repeat = False, auth = True, *args, **kwargs):
 
         # Login first
         if not parameters: parameters = {}
@@ -127,19 +127,20 @@ class NZBVortex(DownloaderBase):
 
         params = tryUrlencode(parameters)
 
-        url = cleanHost(self.conf('host'), ssl = self.conf('ssl')) + 'api/' + call
+        url = cleanHost(self.conf('host')) + 'api/' + call
 
         try:
-            data = self.urlopen('%s?%s' % (url, params), *args, **kwargs)
+            data = self.getJsonData('%s%s' % (url, '?' + params if params else ''), *args, cache_timeout = 0, show_error = False, **kwargs)
 
             if data:
-                return json.loads(data)
-        except URLError as e:
-            if hasattr(e, 'code') and e.code == 403:
+                return data
+        except HTTPError as e:
+            sc = e.response.status_code
+            if sc == 403:
                 # Try login and do again
-                if not repeat:
+                if not is_repeat:
                     self.login()
-                    return self.call(call, parameters = parameters, repeat = True, **kwargs)
+                    return self.call(call, parameters = parameters, is_repeat = True, **kwargs)
 
             log.error('Failed to parsing %s: %s', (self.getName(), traceback.format_exc()))
         except:
@@ -151,13 +152,12 @@ class NZBVortex(DownloaderBase):
 
         if not self.api_level:
 
-            url = cleanHost(self.conf('host')) + 'api/app/apilevel'
-
             try:
-                data = self.urlopen(url, show_error = False)
-                self.api_level = float(json.loads(data).get('apilevel'))
-            except URLError as e:
-                if hasattr(e, 'code') and e.code == 403:
+                data = self.call('app/apilevel', auth = False)
+                self.api_level = float(data.get('apilevel'))
+            except HTTPError as e:
+                sc = e.response.status_code
+                if sc == 403:
                     log.error('This version of NZBVortex isn\'t supported. Please update to 2.8.6 or higher')
                 else:
                     log.error('NZBVortex doesn\'t seem to be running or maybe the remote option isn\'t enabled yet: %s', traceback.format_exc(1))
@@ -169,29 +169,6 @@ class NZBVortex(DownloaderBase):
         return super(NZBVortex, self).isEnabled(manual, data) and self.getApiLevel()
 
 
-class HTTPSConnection(httplib.HTTPSConnection):
-    def __init__(self, *args, **kwargs):
-        httplib.HTTPSConnection.__init__(self, *args, **kwargs)
-
-    def connect(self):
-        sock = socket.create_connection((self.host, self.port), self.timeout)
-        if sys.version_info < (2, 6, 7):
-            if hasattr(self, '_tunnel_host'):
-                self.sock = sock
-                self._tunnel()
-        else:
-            if self._tunnel_host:
-                self.sock = sock
-                self._tunnel()
-
-        self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file, ssl_version = ssl.PROTOCOL_TLSv1)
-
-
-class HTTPSHandler(urllib2.HTTPSHandler):
-    def https_open(self, req):
-        return self.do_open(HTTPSConnection, req)
-
-
 config = [{
     'name': 'nzbvortex',
     'groups': [
@@ -211,20 +188,18 @@ config = [{
                 },
                 {
                     'name': 'host',
-                    'default': 'localhost:4321',
-                    'description': 'Hostname with port. Usually <strong>localhost:4321</strong>',
-                },
-                {
-                    'name': 'ssl',
-                    'default': 1,
-                    'type': 'bool',
-                    'advanced': True,
-                    'description': 'Use HyperText Transfer Protocol Secure, or <strong>https</strong>',
+                    'default': 'https://localhost:4321',
+                    'description': 'Hostname with port. Usually <strong>https://localhost:4321</strong>',
                 },
                 {
                     'name': 'api_key',
                     'label': 'Api Key',
                 },
+                {
+                    'name': 'group',
+                    'label': 'Group',
+                    'description': 'The group CP places the nzb in. Make sure to create it in NZBVortex.',
+                },
                 {
                     'name': 'manual',
                     'default': False,

couchpotato/core/downloaders/rtorrent_.py

@@ -1,18 +1,16 @@
 from base64 import b16encode, b32decode
 from datetime import timedelta
 from hashlib import sha1
-from urlparse import urlparse
+from six.moves import urllib
 import os
 
 from couchpotato.core._base.downloader.main import DownloaderBase, ReleaseDownloadList
-
 from couchpotato.core.event import addEvent
 from couchpotato.core.helpers.encoding import sp
 from couchpotato.core.helpers.variable import cleanHost, splitString
 from couchpotato.core.logger import CPLog
 from bencode import bencode, bdecode
 from rtorrent import RTorrent
-from scandir import scandir
 
 
 log = CPLog(__name__)
@@ -64,7 +62,7 @@ class rTorrent(DownloaderBase):
         if self.conf('ssl') and url.startswith('httprpc://'):
             url = url.replace('httprpc://', 'httprpc+https://')
 
-        parsed = urlparse(url)
+        parsed = urllib.urlparse(url)
 
         # rpc_url is only used on http/https scgi pass-through
         if parsed.scheme in ['http', 'https']:
@@ -154,19 +152,13 @@ class rTorrent(DownloaderBase):
             return False
 
     def getTorrentStatus(self, torrent):
-        if torrent.hashing or torrent.hash_checking or torrent.message:
-            return 'busy'
-
         if not torrent.complete:
             return 'busy'
 
-        if not torrent.open:
-            return 'completed'
-
-        if torrent.state and torrent.active:
+        if torrent.open:
             return 'seeding'
 
-        return 'busy'
+        return 'completed'
 
     def getAllDownloadStatus(self, ids):
         log.debug('Checking rTorrent download status.')
@@ -244,7 +236,7 @@ class rTorrent(DownloaderBase):
             if torrent.is_multi_file() and torrent.directory.endswith(torrent.name):
                 # Remove empty directories bottom up
                 try:
-                    for path, _, _ in scandir.walk(torrent.directory, topdown = False):
+                    for path, _, _ in os.walk(sp(torrent.directory), topdown = False):
                         os.rmdir(path)
                 except OSError:
                     log.info('Directory "%s" contains extra files, unable to remove', torrent.directory)

couchpotato/core/downloaders/synology.py

@@ -90,6 +90,7 @@ class SynologyRPC(object):
 
         self.download_url = 'http://%s:%s/webapi/DownloadStation/task.cgi' % (host, port)
         self.auth_url = 'http://%s:%s/webapi/auth.cgi' % (host, port)
+        self.sid = None
         self.username = username
         self.password = password
         self.destination = destination

couchpotato/core/downloaders/transmission.py

@@ -23,16 +23,14 @@ class Transmission(DownloaderBase):
     log = CPLog(__name__)
     trpc = None
 
-    def connect(self, reconnect = False):
+    def connect(self):
         # Load host from config and split out port.
-        host = cleanHost(self.conf('host'), protocol = False).split(':')
+        host = cleanHost(self.conf('host')).rstrip('/').rsplit(':', 1)
         if not isInt(host[1]):
             log.error('Config properties are not filled in correctly, port is missing.')
             return False
 
-        if not self.trpc or reconnect:
-            self.trpc = TransmissionRPC(host[0], port = host[1], rpc_url = self.conf('rpc_url').strip('/ '), username = self.conf('username'), password = self.conf('password'))
-
+        self.trpc = TransmissionRPC(host[0], port = host[1], rpc_url = self.conf('rpc_url').strip('/ '), username = self.conf('username'), password = self.conf('password'))
         return self.trpc
 
     def download(self, data = None, media = None, filedata = None):
@@ -88,7 +86,7 @@ class Transmission(DownloaderBase):
         return self.downloadReturnId(remote_torrent['torrent-added']['hashString'])
 
     def test(self):
-        if self.connect(True) and self.trpc.get_session():
+        if self.connect() and self.trpc.get_session():
             return True
         return False
 
@@ -164,18 +162,18 @@ class Transmission(DownloaderBase):
 class TransmissionRPC(object):
 
     """TransmissionRPC lite library"""
-    def __init__(self, host = 'localhost', port = 9091, rpc_url = 'transmission', username = None, password = None):
+    def __init__(self, host = 'http://localhost', port = 9091, rpc_url = 'transmission', username = None, password = None):
 
         super(TransmissionRPC, self).__init__()
 
-        self.url = 'http://' + host + ':' + str(port) + '/' + rpc_url + '/rpc'
+        self.url = host + ':' + str(port) + '/' + rpc_url + '/rpc'
         self.tag = 0
         self.session_id = 0
         self.session = {}
         if username and password:
             password_manager = urllib2.HTTPPasswordMgrWithDefaultRealm()
-            password_manager.add_password(realm = None, uri = self.url, user = username, passwd = password)
-            opener = urllib2.build_opener(urllib2.HTTPBasicAuthHandler(password_manager), urllib2.HTTPDigestAuthHandler(password_manager))
+            password_manager.add_password(realm = 'Transmission', uri = self.url, user = username, passwd = password)
+            opener = urllib2.build_opener(urllib2.HTTPBasicAuthHandler(password_manager))
             opener.addheaders = [('User-agent', 'couchpotato-transmission-client/1.0')]
             urllib2.install_opener(opener)
         elif username or password:
@@ -276,8 +274,8 @@ config = [{
                 },
                 {
                     'name': 'host',
-                    'default': 'localhost:9091',
-                    'description': 'Hostname with port. Usually <strong>localhost:9091</strong>',
+                    'default': 'http://localhost:9091',
+                    'description': 'Hostname with port. Usually <strong>http://localhost:9091</strong>',
                 },
                 {
                     'name': 'rpc_url',

couchpotato/core/downloaders/utorrent.py

@@ -168,7 +168,7 @@ class uTorrent(DownloaderBase):
                 status = 'busy'
                 if (torrent[1] & self.status_flags['STARTED'] or torrent[1] & self.status_flags['QUEUED']) and torrent[4] == 1000:
                     status = 'seeding'
-                elif (torrent[1] & self.status_flags['ERROR']):
+                elif torrent[1] & self.status_flags['ERROR']:
                     status = 'failed'
                 elif torrent[4] == 1000:
                     status = 'completed'
@@ -229,7 +229,6 @@ class uTorrentAPI(object):
             password_manager = urllib2.HTTPPasswordMgrWithDefaultRealm()
             password_manager.add_password(realm = None, uri = self.url, user = username, passwd = password)
             self.opener.add_handler(urllib2.HTTPBasicAuthHandler(password_manager))
-            self.opener.add_handler(urllib2.HTTPDigestAuthHandler(password_manager))
         elif username or password:
             log.debug('User or password missing, not using authentication.')
         self.token = self.get_token()

couchpotato/core/event.py

@@ -90,7 +90,7 @@ def fireEvent(name, *args, **kwargs):
 
         else:
 
-            e = Event(name = name, threads = 10, exc_info = True, traceback = True, lock = threading.RLock())
+            e = Event(name = name, threads = 10, exc_info = True, traceback = True)
 
             for event in events[name]:
                 e.handle(event['handler'], priority = event['priority'])
@@ -102,7 +102,7 @@ def fireEvent(name, *args, **kwargs):
             # Fire
             result = e(*args, **kwargs)
 
-        result_keys = result.keys()
+        result_keys = list(result.keys())
         result_keys.sort(key = natsortKey)
 
         if options['single'] and not options['merge']:

couchpotato/core/helpers/database.py

@@ -0,0 +1,26 @@
+
+from six import PY2
+
+
+if PY2:
+    from CodernityDB.database_super_thread_safe import SuperThreadSafeDatabase
+    from CodernityDB.index import IndexException, IndexConflict, IndexNotFoundException
+    from CodernityDB.database import RecordNotFound, RecordDeleted
+    from CodernityDB.hash_index import HashIndex
+    from CodernityDB.tree_index import MultiTreeBasedIndex, TreeBasedIndex
+else:
+    from CodernityDB3.database_super_thread_safe import SuperThreadSafeDatabase
+    from CodernityDB3.index import IndexException, IndexConflict, IndexNotFoundException
+    from CodernityDB3.database import RecordNotFound, RecordDeleted
+    from CodernityDB3.hash_index import HashIndex
+    from CodernityDB3.tree_index import MultiTreeBasedIndex, TreeBasedIndex
+
+SuperThreadSafeDatabase = SuperThreadSafeDatabase
+IndexException = IndexException
+IndexNotFoundException = IndexNotFoundException
+IndexConflict = IndexConflict
+RecordNotFound = RecordNotFound
+HashIndex = HashIndex
+MultiTreeBasedIndex = MultiTreeBasedIndex
+TreeBasedIndex = TreeBasedIndex
+RecordDeleted = RecordDeleted

couchpotato/core/helpers/encoding.py

@@ -1,10 +1,11 @@
 from string import ascii_letters, digits
-from urllib import quote_plus
 import os
 import re
 import traceback
 import unicodedata
 
+from chardet import detect
+from six.moves import urllib
 from couchpotato.core.logger import CPLog
 import six
 
@@ -15,7 +16,7 @@ log = CPLog(__name__)
 def toSafeString(original):
     valid_chars = "-_.() %s%s" % (ascii_letters, digits)
     cleaned_filename = unicodedata.normalize('NFKD', toUnicode(original)).encode('ASCII', 'ignore')
-    valid_string = ''.join(c for c in cleaned_filename if c in valid_chars)
+    valid_string = ''.join(list(six.unichr(c) for c in cleaned_filename if six.unichr(c) in valid_chars))
     return ' '.join(valid_string.split())
 
 
@@ -28,13 +29,16 @@ def simplifyString(original):
 
 def toUnicode(original, *args):
     try:
-        if isinstance(original, unicode):
+        if isinstance(original, six.text_type):
             return original
         else:
             try:
                 return six.text_type(original, *args)
             except:
                 try:
+                    detected = detect(original)
+                    if detected.get('encoding') == 'utf-8':
+                        return original.decode('utf-8')
                     return ek(original, *args)
                 except:
                     raise
@@ -43,16 +47,35 @@ def toUnicode(original, *args):
         ascii_text = str(original).encode('string_escape')
         return toUnicode(ascii_text)
 
+def toUTF8(original):
+    try:
+        if isinstance(original, six.binary_type) and len(original) > 0:
+            # Try to detect
+            detected = detect(original)
+            return original.decode(detected.get('encoding')).encode('utf-8')
+        else:
+            return original
+    except:
+        #log.error('Failed encoding to UTF8: %s', traceback.format_exc())
+        raise
 
 def ss(original, *args):
 
     u_original = toUnicode(original, *args)
     try:
-        from couchpotato.environment import Env
-        return u_original.encode(Env.get('encoding'))
+        if isinstance(u_original, six.text_type):
+            u_original = u_original.encode('unicode_escape')
+        else:
+            u_original = u_original
+
+        return six.u(u_original)
     except Exception as e:
         log.debug('Failed ss encoding char, force UTF8: %s', e)
-        return u_original.encode('UTF-8')
+        try:
+            from couchpotato.environment import Env
+            return u_original.encode(Env.get('encoding'), 'replace')
+        except:
+            return u_original.encode('utf-8', 'replace')
 
 
 def sp(path, *args):
@@ -65,7 +88,7 @@ def sp(path, *args):
     if os.path.sep == '/' and '\\' in path:
         path = '/' + path.replace(':', '').replace('\\', '/')
 
-    path = os.path.normpath(ss(path, *args))
+    path = os.path.normpath(path)
 
     # Remove any trailing path separators
     if path != os.path.sep:
@@ -78,7 +101,7 @@ def sp(path, *args):
     # Replace *NIX ambiguous '//' at the beginning of a path with '/' (crashes guessit)
     path = re.sub('^//', '/', path)
 
-    return toUnicode(path)
+    return path
 
 
 def ek(original, *args):
@@ -107,14 +130,15 @@ def stripAccents(s):
 def tryUrlencode(s):
     new = six.u('')
     if isinstance(s, dict):
-        for key, value in s.items():
+        for key, value in list(s.items()):
             new += six.u('&%s=%s') % (key, tryUrlencode(value))
 
         return new[1:]
     else:
         for letter in ss(s):
+            letter = six.unichr(letter)
             try:
-                new += quote_plus(letter)
+                new += urllib.parse.quote_plus(letter)
             except:
                 new += letter
 

couchpotato/core/helpers/path.py

@@ -0,0 +1,51 @@
+import os
+from chardet import detect
+from couchpotato import Env
+
+fs_enc = Env.get('fs_encoding')
+
+
+def list_dir(path, full_path = True):
+    """
+    List directory don't error when it doesn't exist
+    """
+
+    path = unicode_path(path)
+
+    if os.path.isdir(path):
+        for f in os.listdir(path):
+            if full_path:
+                yield join(path, f)
+            else:
+                yield f
+
+
+def join(*args):
+    """
+    Join path, encode properly before joining
+    """
+
+    return os.path.join(*[safe(x) for x in args])
+
+
+def unicode_path(path):
+    """
+    Convert back to unicode
+    :param path: path string
+    """
+
+    if isinstance(path, str):
+        detected = detect(path)
+        print detected
+        path = path.decode(detected.get('encoding'))
+        path = path.decode('unicode_escape')
+
+    return path
+
+
+def safe(path):
+
+    if isinstance(path, unicode):
+        return path.encode('unicode_escape')
+
+    return path

couchpotato/core/helpers/py3.py

@@ -0,0 +1,2 @@
+class NotSupported(Exception):
+    pass

couchpotato/core/helpers/request.py

@@ -1,7 +1,7 @@
-from urllib import unquote
 import re
 
 from couchpotato.core.helpers.encoding import toUnicode
+from six.moves import urllib
 from couchpotato.core.helpers.variable import natsortKey
 
 
@@ -10,7 +10,7 @@ def getParams(params):
     reg = re.compile('^[a-z0-9_\.]+$')
 
     # Sort keys
-    param_keys = params.keys()
+    param_keys = list(params.keys())
     param_keys.sort(key = natsortKey)
 
     temp = {}
@@ -28,7 +28,7 @@ def getParams(params):
 
             for item in nested:
                 if item is nested[-1]:
-                    current[item] = toUnicode(unquote(value))
+                    current[item] = toUnicode(urllib.parse.unquote(value))
                 else:
                     try:
                         current[item]
@@ -37,7 +37,7 @@ def getParams(params):
 
                     current = current[item]
         else:
-            temp[param] = toUnicode(unquote(value))
+            temp[param] = toUnicode(urllib.parse.unquote(value))
             if temp[param].lower() in ['true', 'false']:
                 temp[param] = temp[param].lower() != 'false'
 

couchpotato/core/helpers/variable.py

@@ -1,4 +1,5 @@
 import collections
+import ctypes
 import hashlib
 import os
 import platform
@@ -6,8 +7,9 @@ import random
 import re
 import string
 import sys
+import traceback
 
-from couchpotato.core.helpers.encoding import simplifyString, toSafeString, ss
+from couchpotato.core.helpers.encoding import simplifyString, toSafeString, ss, sp
 from couchpotato.core.logger import CPLog
 import six
 from six.moves import map, zip, filter
@@ -39,11 +41,11 @@ def symlink(src, dst):
 def getUserDir():
     try:
         import pwd
-        os.environ['HOME'] = pwd.getpwuid(os.geteuid()).pw_dir
+        os.environ['HOME'] = sp(pwd.getpwuid(os.geteuid()).pw_dir)
     except:
         pass
 
-    return os.path.expanduser('~')
+    return sp(os.path.expanduser('~'))
 
 
 def getDownloadDir():
@@ -290,9 +292,14 @@ def dictIsSubset(a, b):
     return all([k in b and b[k] == v for k, v in a.items()])
 
 
+# Returns True if sub_folder is the same as or inside base_folder
 def isSubFolder(sub_folder, base_folder):
-    # Returns True if sub_folder is the same as or inside base_folder
-    return base_folder and sub_folder and ss(os.path.normpath(base_folder).rstrip(os.path.sep) + os.path.sep) in ss(os.path.normpath(sub_folder).rstrip(os.path.sep) + os.path.sep)
+    if base_folder and sub_folder:
+        base = sp(os.path.realpath(base_folder)) + os.path.sep
+        subfolder = sp(os.path.realpath(sub_folder)) + os.path.sep
+        return os.path.commonprefix([subfolder, base]) == base
+
+    return False
 
 
 # From SABNZBD
@@ -307,3 +314,99 @@ def scanForPassword(name):
 
     if m:
         return m.group(1).strip('. '), m.group(2).strip()
+
+
+under_pat = re.compile(r'_([a-z])')
+
+def underscoreToCamel(name):
+    return under_pat.sub(lambda x: x.group(1).upper(), name)
+
+
+def removePyc(folder, only_excess = True, show_logs = True):
+
+    folder = sp(folder)
+
+    for root, dirs, files in os.walk(folder):
+
+        pyc_files = filter(lambda filename: filename.endswith('.pyc'), files)
+        py_files = set(filter(lambda filename: filename.endswith('.py'), files))
+        excess_pyc_files = filter(lambda pyc_filename: pyc_filename[:-1] not in py_files, pyc_files) if only_excess else pyc_files
+
+        for excess_pyc_file in excess_pyc_files:
+            full_path = os.path.join(root, excess_pyc_file)
+            if show_logs: log.debug('Removing old PYC file: %s', full_path)
+            try:
+                os.remove(full_path)
+            except:
+                log.error('Couldn\'t remove %s: %s', (full_path, traceback.format_exc()))
+
+        for dir_name in dirs:
+            full_path = os.path.join(root, dir_name)
+            if len(os.listdir(full_path)) == 0:
+                try:
+                    os.rmdir(full_path)
+                except:
+                    log.error('Couldn\'t remove empty directory %s: %s', (full_path, traceback.format_exc()))
+
+
+def getFreeSpace(directories):
+
+    single = not isinstance(directories, (tuple, list))
+    if single:
+        directories = [directories]
+
+    free_space = {}
+    for folder in directories:
+
+        size = None
+        if os.path.isdir(folder):
+            if os.name == 'nt':
+                _, total, free = ctypes.c_ulonglong(), ctypes.c_ulonglong(), \
+                                   ctypes.c_ulonglong()
+                if sys.version_info >= (3,) or isinstance(folder, unicode):
+                    fun = ctypes.windll.kernel32.GetDiskFreeSpaceExW #@UndefinedVariable
+                else:
+                    fun = ctypes.windll.kernel32.GetDiskFreeSpaceExA #@UndefinedVariable
+                ret = fun(folder, ctypes.byref(_), ctypes.byref(total), ctypes.byref(free))
+                if ret == 0:
+                    raise ctypes.WinError()
+                return [total.value, free.value]
+            else:
+                s = os.statvfs(folder)
+                size = [s.f_blocks * s.f_frsize / (1024 * 1024), (s.f_bavail * s.f_frsize) / (1024 * 1024)]
+
+        if single: return size
+
+        free_space[folder] = size
+
+    return free_space
+
+
+def getSize(paths):
+
+    single = not isinstance(paths, (tuple, list))
+    if single:
+        paths = [paths]
+
+    total_size = 0
+    for path in paths:
+        path = sp(path)
+
+        if os.path.isdir(path):
+            total_size = 0
+            for dirpath, _, filenames in os.walk(path):
+                for f in filenames:
+                    total_size += os.path.getsize(sp(os.path.join(dirpath, f)))
+
+        elif os.path.isfile(path):
+            total_size += os.path.getsize(path)
+
+    return total_size / 1048576 # MB
+
+
+def find(func, iterable):
+    for item in iterable:
+        if func(item):
+            return item
+
+    return None

couchpotato/core/loader.py

@@ -3,6 +3,7 @@ import sys
 import traceback
 
 from couchpotato.core.event import fireEvent
+from couchpotato.core.helpers.py3 import NotSupported
 from couchpotato.core.logger import CPLog
 from importhelper import import_module
 import six
@@ -131,7 +132,7 @@ class Loader(object):
             return False
         try:
             # Load single file plugin
-            if isinstance(module.autoload, (str, unicode)):
+            if isinstance(module.autoload, (six.string_types, six.text_type)):
                 getattr(module, module.autoload)()
             # Load folder plugin
             else:
@@ -162,6 +163,8 @@ class Loader(object):
     def loadModule(self, name):
         try:
             return import_module(name)
+        except NotSupported:
+            log.error('Module "%s" is not supported in Python 3', name)
         except ImportError:
             log.debug('Skip loading module plugin %s: %s', (name, traceback.format_exc()))
             return None

couchpotato/core/logger.py

@@ -1,5 +1,6 @@
 import logging
 import re
+import traceback
 
 
 class CPLog(object):
@@ -25,6 +26,12 @@ class CPLog(object):
             self.Env = Env
             self.is_develop = Env.get('dev')
 
+            from couchpotato.core.event import addEvent
+            addEvent('app.after_shutdown', self.close)
+
+    def close(self, *args, **kwargs):
+        logging.shutdown()
+
     def info(self, msg, replace_tuple = ()):
         self.logger.info(self.addContext(msg, replace_tuple))
 
@@ -48,20 +55,19 @@ class CPLog(object):
 
     def safeMessage(self, msg, replace_tuple = ()):
 
-        from couchpotato.core.helpers.encoding import ss, toUnicode
+        from couchpotato.core.helpers.encoding import ss, toUTF8
 
-        msg = ss(msg)
+        msg = toUTF8(msg)
 
         try:
-            msg = msg % replace_tuple
+            if isinstance(replace_tuple, tuple):
+                msg = msg % tuple([toUTF8(x) for x in list(replace_tuple)])
+            elif isinstance(replace_tuple, dict):
+                msg = msg % dict((k, toUTF8(v)) for k, v in replace_tuple.iteritems())
+            else:
+                msg = msg % toUTF8(replace_tuple)
         except:
-            try:
-                if isinstance(replace_tuple, tuple):
-                    msg = msg % tuple([ss(x) for x in list(replace_tuple)])
-                else:
-                    msg = msg % ss(replace_tuple)
-            except Exception as e:
-                self.logger.error('Failed encoding stuff to log "%s": %s' % (msg, e))
+            self.logger.error('Failed encoding stuff to log "%s": %s' % (msg, traceback.format_exc()))
 
         self.setup()
         if not self.is_develop:
@@ -78,4 +84,4 @@ class CPLog(object):
             except:
                 pass
 
-        return toUnicode(msg)
+        return toUTF8(msg)

couchpotato/core/media/__init__.py

@@ -1,7 +1,7 @@
 import os
 import traceback
 
-from couchpotato import get_db, CPLog
+from couchpotato import CPLog
 from couchpotato.core.event import addEvent, fireEvent, fireEventAsync
 from couchpotato.core.helpers.encoding import toUnicode
 from couchpotato.core.plugins.base import Plugin
@@ -25,11 +25,10 @@ class MediaBase(Plugin):
 
         def onComplete():
             try:
-                db = get_db()
                 media = fireEvent('media.get', media_id, single = True)
-                event_name = '%s.searcher.single' % media.get('type')
-
-                fireEventAsync(event_name, media, on_complete = self.createNotifyFront(media_id))
+                if media:
+                    event_name = '%s.searcher.single' % media.get('type')
+                    fireEventAsync(event_name, media, on_complete = self.createNotifyFront(media_id), manual = True)
             except:
                 log.error('Failed creating onComplete: %s', traceback.format_exc())
 
@@ -40,9 +39,9 @@ class MediaBase(Plugin):
         def notifyFront():
             try:
                 media = fireEvent('media.get', media_id, single = True)
-                event_name = '%s.update' % media.get('type')
-
-                fireEvent('notify.frontend', type = event_name, data = media)
+                if media:
+                    event_name = '%s.update' % media.get('type')
+                    fireEvent('notify.frontend', type = event_name, data = media)
             except:
                 log.error('Failed creating onComplete: %s', traceback.format_exc())
 
@@ -66,10 +65,13 @@ class MediaBase(Plugin):
 
         return def_title or 'UNKNOWN'
 
-    def getPoster(self, image_urls, existing_files):
-        image_type = 'poster'
+    def getPoster(self, media, image_urls):
+        if 'files' not in media:
+            media['files'] = {}
 
-        # Remove non-existing files
+        existing_files = media['files']
+
+        image_type = 'poster'
         file_type = 'image_%s' % image_type
 
         # Make existing unique
@@ -87,13 +89,13 @@ class MediaBase(Plugin):
 
         # Loop over type
         for image in image_urls.get(image_type, []):
-            if not isinstance(image, (str, unicode)):
+            if not isinstance(image, six.string_types):
                 continue
 
             if file_type not in existing_files or len(existing_files.get(file_type, [])) == 0:
                 file_path = fireEvent('file.download', url = image, single = True)
                 if file_path:
-                    existing_files[file_type] = [file_path]
+                    existing_files[file_type] = [toUnicode(file_path)]
                     break
             else:
                 break

couchpotato/core/media/_base/library/main.py

@@ -1,10 +1,47 @@
+from couchpotato import get_db
+from couchpotato.api import addApiView
 from couchpotato.core.event import addEvent, fireEvent
+from couchpotato.core.logger import CPLog
 from couchpotato.core.media._base.library.base import LibraryBase
 
+log = CPLog(__name__)
+
 
 class Library(LibraryBase):
     def __init__(self):
         addEvent('library.title', self.title)
+        addEvent('library.related', self.related)
+        addEvent('library.tree', self.tree)
+
+        addEvent('library.root', self.root)
+
+        addApiView('library.query', self.queryView)
+        addApiView('library.related', self.relatedView)
+        addApiView('library.tree', self.treeView)
+
+    def queryView(self, media_id, **kwargs):
+        db = get_db()
+        media = db.get('id', media_id)
+
+        return {
+            'result': fireEvent('library.query', media, single = True)
+        }
+
+    def relatedView(self, media_id, **kwargs):
+        db = get_db()
+        media = db.get('id', media_id)
+
+        return {
+            'result': fireEvent('library.related', media, single = True)
+        }
+
+    def treeView(self, media_id, **kwargs):
+        db = get_db()
+        media = db.get('id', media_id)
+
+        return {
+            'result': fireEvent('library.tree', media, single = True)
+        }
 
     def title(self, library):
         return fireEvent(
@@ -16,3 +53,76 @@ class Library(LibraryBase):
             include_identifier = False,
             single = True
         )
+
+    def related(self, media):
+        result = {self.key(media['type']): media}
+
+        db = get_db()
+        cur = media
+
+        while cur and cur.get('parent_id'):
+            cur = db.get('id', cur['parent_id'])
+
+            result[self.key(cur['type'])] = cur
+
+        children = db.get_many('media_children', media['_id'], with_doc = True)
+
+        for item in children:
+            key = self.key(item['doc']['type']) + 's'
+
+            if key not in result:
+                result[key] = []
+
+            result[key].append(item['doc'])
+
+        return result
+
+    def root(self, media):
+        db = get_db()
+        cur = media
+
+        while cur and cur.get('parent_id'):
+            cur = db.get('id', cur['parent_id'])
+
+        return cur
+
+    def tree(self, media = None, media_id = None):
+        db = get_db()
+
+        if media:
+            result = media
+        elif media_id:
+            result = db.get('id', media_id, with_doc = True)
+        else:
+            return None
+
+        # Find children
+        items = db.get_many('media_children', result['_id'], with_doc = True)
+        keys = []
+
+        # Build children arrays
+        for item in items:
+            key = self.key(item['doc']['type']) + 's'
+
+            if key not in result:
+                result[key] = {}
+            elif type(result[key]) is not dict:
+                result[key] = {}
+
+            if key not in keys:
+                keys.append(key)
+
+            result[key][item['_id']] = fireEvent('library.tree', item['doc'], single = True)
+
+        # Unique children
+        for key in keys:
+            result[key] = result[key].values()
+
+        # Include releases
+        result['releases'] = fireEvent('release.for_media', result['_id'], single = True)
+
+        return result
+
+    def key(self, media_type):
+        parts = media_type.split('.')
+        return parts[-1]

couchpotato/core/media/_base/matcher/main.py

@@ -40,7 +40,7 @@ class Matcher(MatcherBase):
         return False
 
     def correctTitle(self, chain, media):
-        root_library = media['library']['root_library']
+        root = fireEvent('library.root', media, single = True)
 
         if 'show_name' not in chain.info or not len(chain.info['show_name']):
             log.info('Wrong: missing show name in parsed result')
@@ -50,10 +50,10 @@ class Matcher(MatcherBase):
         chain_words = [x.lower() for x in chain.info['show_name']]
 
         # Build a list of possible titles of the media we are searching for
-        titles = root_library['info']['titles']
+        titles = root['info']['titles']
 
         # Add year suffix titles (will result in ['<name_one>', '<name_one> <suffix_one>', '<name_two>', ...])
-        suffixes = [None, root_library['info']['year']]
+        suffixes = [None, root['info']['year']]
 
         titles = [
             title + ((' %s' % suffix) if suffix else '')

couchpotato/core/media/_base/media/index.py

@@ -1,14 +1,14 @@
 from string import ascii_letters
 from hashlib import md5
+from couchpotato.core.helpers.database import MultiTreeBasedIndex, TreeBasedIndex
 
-from CodernityDB.tree_index import MultiTreeBasedIndex, TreeBasedIndex
 from couchpotato.core.helpers.encoding import toUnicode, simplifyString
 
 
 class MediaIndex(MultiTreeBasedIndex):
     _version = 3
 
-    custom_header = """from CodernityDB.tree_index import MultiTreeBasedIndex"""
+    custom_header = """from couchpotato.core.helpers.database import MultiTreeBasedIndex"""
 
     def __init__(self, *args, **kwargs):
         kwargs['key_format'] = '32s'
@@ -62,10 +62,11 @@ class MediaTypeIndex(TreeBasedIndex):
 
 
 class TitleSearchIndex(MultiTreeBasedIndex):
-    _version = 1
+    _version = 2
 
-    custom_header = """from CodernityDB.tree_index import MultiTreeBasedIndex
-from itertools import izip
+    custom_header = """from couchpotato.core.helpers.database import MultiTreeBasedIndex
+try: from itertools import izip
+except: izip = zip
 from couchpotato.core.helpers.encoding import simplifyString"""
 
     def __init__(self, *args, **kwargs):
@@ -99,9 +100,9 @@ from couchpotato.core.helpers.encoding import simplifyString"""
 
 
 class TitleIndex(TreeBasedIndex):
-    _version = 2
+    _version = 4
 
-    custom_header = """from CodernityDB.tree_index import TreeBasedIndex
+    custom_header = """from couchpotato.core.helpers.database import TreeBasedIndex
 from string import ascii_letters
 from couchpotato.core.helpers.encoding import toUnicode, simplifyString"""
 
@@ -123,18 +124,18 @@ from couchpotato.core.helpers.encoding import toUnicode, simplifyString"""
         nr_prefix = '' if title and len(title) > 0 and title[0] in ascii_letters else '#'
         title = simplifyString(title)
 
-        for prefix in ['the ']:
+        for prefix in ['the ', 'an ', 'a ']:
             if prefix == title[:len(prefix)]:
                 title = title[len(prefix):]
                 break
 
-        return str(nr_prefix + title).ljust(32, '_')[:32]
+        return str(nr_prefix + title).ljust(32, ' ')[:32]
 
 
 class StartsWithIndex(TreeBasedIndex):
-    _version = 2
+    _version = 3
 
-    custom_header = """from CodernityDB.tree_index import TreeBasedIndex
+    custom_header = """from couchpotato.core.helpers.database import TreeBasedIndex
 from string import ascii_letters
 from couchpotato.core.helpers.encoding import toUnicode, simplifyString"""
 
@@ -153,7 +154,7 @@ from couchpotato.core.helpers.encoding import toUnicode, simplifyString"""
         title = toUnicode(title)
         title = simplifyString(title)
 
-        for prefix in ['the ']:
+        for prefix in ['the ', 'an ', 'a ']:
             if prefix == title[:len(prefix)]:
                 title = title[len(prefix):]
                 break
@@ -176,3 +177,24 @@ class MediaChildrenIndex(TreeBasedIndex):
         if data.get('_t') == 'media' and data.get('parent_id'):
             return data.get('parent_id'), None
 
+
+class MediaTagIndex(MultiTreeBasedIndex):
+    _version = 2
+
+    custom_header = """from couchpotato.core.helpers.database import MultiTreeBasedIndex"""
+
+    def __init__(self, *args, **kwargs):
+        kwargs['key_format'] = '32s'
+        super(MediaTagIndex, self).__init__(*args, **kwargs)
+
+    def make_key_value(self, data):
+        if data.get('_t') == 'media' and data.get('tags') and len(data.get('tags', [])) > 0:
+
+            tags = set()
+            for tag in data.get('tags', []):
+                tags.add(self.make_key(tag))
+
+            return list(tags), None
+
+    def make_key(self, key):
+        return md5(key).hexdigest()

couchpotato/core/media/_base/media/main.py

@@ -1,14 +1,18 @@
+from datetime import timedelta
+import time
 import traceback
 from string import ascii_lowercase
 
 from couchpotato import tryInt, get_db
 from couchpotato.api import addApiView
 from couchpotato.core.event import fireEvent, fireEventAsync, addEvent
+from couchpotato.core.helpers.database import RecordNotFound, RecordDeleted
 from couchpotato.core.helpers.encoding import toUnicode
 from couchpotato.core.helpers.variable import splitString, getImdb, getTitle
 from couchpotato.core.logger import CPLog
 from couchpotato.core.media import MediaBase
-from .index import MediaIndex, MediaStatusIndex, MediaTypeIndex, TitleSearchIndex, TitleIndex, StartsWithIndex, MediaChildrenIndex
+from .index import MediaIndex, MediaStatusIndex, MediaTypeIndex, TitleSearchIndex, TitleIndex, StartsWithIndex, MediaChildrenIndex, MediaTagIndex
+import six
 
 
 log = CPLog(__name__)
@@ -20,6 +24,7 @@ class MediaPlugin(MediaBase):
         'media': MediaIndex,
         'media_search_title': TitleSearchIndex,
         'media_status': MediaStatusIndex,
+        'media_tag': MediaTagIndex,
         'media_by_type': MediaTypeIndex,
         'media_title': TitleIndex,
         'media_startswith': StartsWithIndex,
@@ -39,15 +44,15 @@ class MediaPlugin(MediaBase):
             'desc': 'List media',
             'params': {
                 'type': {'type': 'string', 'desc': 'Media type to filter on.'},
-                'status': {'type': 'array or csv', 'desc': 'Filter movie by status. Example:"active,done"'},
-                'release_status': {'type': 'array or csv', 'desc': 'Filter movie by status of its releases. Example:"snatched,available"'},
-                'limit_offset': {'desc': 'Limit and offset the movie list. Examples: "50" or "50,30"'},
-                'starts_with': {'desc': 'Starts with these characters. Example: "a" returns all movies starting with the letter "a"'},
-                'search': {'desc': 'Search movie title'},
+                'status': {'type': 'array or csv', 'desc': 'Filter media by status. Example:"active,done"'},
+                'release_status': {'type': 'array or csv', 'desc': 'Filter media by status of its releases. Example:"snatched,available"'},
+                'limit_offset': {'desc': 'Limit and offset the media list. Examples: "50" or "50,30"'},
+                'starts_with': {'desc': 'Starts with these characters. Example: "a" returns all media starting with the letter "a"'},
+                'search': {'desc': 'Search media title'},
             },
             'return': {'type': 'object', 'example': """{
     'success': True,
-    'empty': bool, any movies returned or not,
+    'empty': bool, any media returned or not,
     'media': array, media found,
 }"""}
         })
@@ -73,6 +78,7 @@ class MediaPlugin(MediaBase):
         addEvent('app.load', self.addSingleListView, priority = 100)
         addEvent('app.load', self.addSingleCharView, priority = 100)
         addEvent('app.load', self.addSingleDeleteView, priority = 100)
+        addEvent('app.load', self.cleanupFaults)
 
         addEvent('media.get', self.get)
         addEvent('media.with_status', self.withStatus)
@@ -80,6 +86,20 @@ class MediaPlugin(MediaBase):
         addEvent('media.list', self.list)
         addEvent('media.delete', self.delete)
         addEvent('media.restatus', self.restatus)
+        addEvent('media.tag', self.tag)
+        addEvent('media.untag', self.unTag)
+
+    # Wrongly tagged media files
+    def cleanupFaults(self):
+        medias = fireEvent('media.with_status', 'ignored', single = True) or []
+
+        db = get_db()
+        for media in medias:
+            try:
+                media['status'] = 'done'
+                db.update(media)
+            except:
+                pass
 
     def refresh(self, id = '', **kwargs):
         handlers = []
@@ -102,13 +122,12 @@ class MediaPlugin(MediaBase):
 
         try:
             media = get_db().get('id', media_id)
-            event = '%s.update_info' % media.get('type')
+            event = '%s.update' % media.get('type')
 
             def handler():
                 fireEvent(event, media_id = media_id, on_complete = self.createOnComplete(media_id))
 
-            if handler:
-                return handler
+            return handler
 
         except:
             log.error('Refresh handler for non existing media: %s', traceback.format_exc())
@@ -120,25 +139,30 @@ class MediaPlugin(MediaBase):
 
     def get(self, media_id):
 
-        db = get_db()
+        try:
+            db = get_db()
 
-        imdb_id = getImdb(str(media_id))
+            imdb_id = getImdb(str(media_id))
 
-        media = None
-        if imdb_id:
-            media = db.get('media', 'imdb-%s' % imdb_id, with_doc = True)['doc']
-        else:
-            media = db.get('id', media_id)
+            if imdb_id:
+                media = db.get('media', 'imdb-%s' % imdb_id, with_doc = True)['doc']
+            else:
+                media = db.get('id', media_id)
 
-        if media:
+            if media:
 
-            # Attach category
-            try: media['category'] = db.get('id', media.get('category_id'))
-            except: pass
+                # Attach category
+                try: media['category'] = db.get('id', media.get('category_id'))
+                except: pass
 
-            media['releases'] = fireEvent('release.for_media', media['_id'], single = True)
+                media['releases'] = fireEvent('release.for_media', media['_id'], single = True)
 
-        return media
+            return media
+
+        except (RecordNotFound, RecordDeleted):
+            log.error('Media with id "%s" not found', media_id)
+        except:
+            raise
 
     def getView(self, id = None, **kwargs):
 
@@ -149,30 +173,45 @@ class MediaPlugin(MediaBase):
             'media': media,
         }
 
-    def withStatus(self, status, with_doc = True):
+    def withStatus(self, status, types = None, with_doc = True):
 
         db = get_db()
 
+        if types and not isinstance(types, (list, tuple)):
+            types = [types]
+
         status = list(status if isinstance(status, (list, tuple)) else [status])
 
         for s in status:
-            for ms in db.get_many('media_status', s, with_doc = with_doc):
-                yield ms['doc'] if with_doc else ms
+            for ms in db.get_many('media_status', s):
+                if with_doc:
+                    try:
+                        doc = db.get('id', ms['_id'])
+
+                        if types and doc.get('type') not in types:
+                            continue
+
+                        yield doc
+                    except (RecordDeleted, RecordNotFound):
+                        log.debug('Record not found, skipping: %s', ms['_id'])
+                    except (ValueError, EOFError):
+                        fireEvent('database.delete_corrupted', ms.get('_id'), traceback_error = traceback.format_exc(0))
+                else:
+                    yield ms
 
     def withIdentifiers(self, identifiers, with_doc = False):
-
         db = get_db()
 
         for x in identifiers:
             try:
-                media = db.get('media', '%s-%s' % (x, identifiers[x]), with_doc = with_doc)
-                return media
+                return db.get('media', '%s-%s' % (x, identifiers[x]), with_doc = with_doc)
             except:
                 pass
 
         log.debug('No media found with identifiers: %s', identifiers)
+        return False
 
-    def list(self, types = None, status = None, release_status = None, status_or = False, limit_offset = None, starts_with = None, search = None):
+    def list(self, types = None, status = None, release_status = None, status_or = False, limit_offset = None, with_tags = None, starts_with = None, search = None):
 
         db = get_db()
 
@@ -183,6 +222,8 @@ class MediaPlugin(MediaBase):
             release_status = [release_status]
         if types and not isinstance(types, (list, tuple)):
             types = [types]
+        if with_tags and not isinstance(with_tags, (list, tuple)):
+            with_tags = [with_tags]
 
         # query media ids
         if types:
@@ -209,11 +250,17 @@ class MediaPlugin(MediaBase):
 
         # Add search filters
         if starts_with:
-            filter_by['starts_with'] = set()
             starts_with = toUnicode(starts_with.lower())[0]
             starts_with = starts_with if starts_with in ascii_lowercase else '#'
             filter_by['starts_with'] = [x['_id'] for x in db.get_many('media_startswith', starts_with)]
 
+        # Add tag filter
+        if with_tags:
+            filter_by['with_tags'] = set()
+            for tag in with_tags:
+                for x in db.get_many('media_tag', tag):
+                    filter_by['with_tags'].add(x['_id'])
+
         # Filter with search query
         if search:
             filter_by['search'] = [x['_id'] for x in db.get_many('media_search_title', search)]
@@ -234,7 +281,7 @@ class MediaPlugin(MediaBase):
         offset = 0
         limit = -1
         if limit_offset:
-            splt = splitString(limit_offset) if isinstance(limit_offset, (str, unicode)) else limit_offset
+            splt = splitString(limit_offset) if isinstance(limit_offset, six.string_types) else limit_offset
             limit = tryInt(splt[0])
             offset = tryInt(0 if len(splt) is 1 else splt[1])
 
@@ -249,6 +296,10 @@ class MediaPlugin(MediaBase):
 
             media = fireEvent('media.get', media_id, single = True)
 
+            # Skip if no media has been found
+            if not media:
+                continue
+
             # Merge releases with movie dict
             medias.append(media)
 
@@ -266,6 +317,7 @@ class MediaPlugin(MediaBase):
             release_status = splitString(kwargs.get('release_status')),
             status_or = kwargs.get('status_or') is not None,
             limit_offset = kwargs.get('limit_offset'),
+            with_tags = splitString(kwargs.get('with_tags')),
             starts_with = kwargs.get('starts_with'),
             search = kwargs.get('search')
         )
@@ -280,9 +332,22 @@ class MediaPlugin(MediaBase):
     def addSingleListView(self):
 
         for media_type in fireEvent('media.types', merge = True):
-            def tempList(*args, **kwargs):
-                return self.listView(types = media_type, **kwargs)
-            addApiView('%s.list' % media_type, tempList)
+            tempList = lambda *args, **kwargs : self.listView(type = media_type, **kwargs)
+            addApiView('%s.list' % media_type, tempList, docs = {
+                'desc': 'List media',
+                'params': {
+                    'status': {'type': 'array or csv', 'desc': 'Filter ' + media_type + ' by status. Example:"active,done"'},
+                    'release_status': {'type': 'array or csv', 'desc': 'Filter ' + media_type + ' by status of its releases. Example:"snatched,available"'},
+                    'limit_offset': {'desc': 'Limit and offset the ' + media_type + ' list. Examples: "50" or "50,30"'},
+                    'starts_with': {'desc': 'Starts with these characters. Example: "a" returns all ' + media_type + 's starting with the letter "a"'},
+                    'search': {'desc': 'Search ' + media_type + ' title'},
+                },
+                'return': {'type': 'object', 'example': """{
+        'success': True,
+        'empty': bool, any """ + media_type + """s returned or not,
+        'media': array, media found,
+    }"""}
+            })
 
     def availableChars(self, types = None, status = None, release_status = None):
 
@@ -328,7 +393,7 @@ class MediaPlugin(MediaBase):
             if x['_id'] in media_ids:
                 chars.add(x['key'])
 
-            if len(chars) == 25:
+            if len(chars) == 27:
                 break
 
         return list(chars)
@@ -349,8 +414,7 @@ class MediaPlugin(MediaBase):
     def addSingleCharView(self):
 
         for media_type in fireEvent('media.types', merge = True):
-            def tempChar(*args, **kwargs):
-                return self.charView(types = media_type, **kwargs)
+            tempChar = lambda *args, **kwargs : self.charView(type = media_type, **kwargs)
             addApiView('%s.available_chars' % media_type, tempChar)
 
     def delete(self, media_id, delete_from = None):
@@ -361,13 +425,18 @@ class MediaPlugin(MediaBase):
             media = db.get('id', media_id)
             if media:
                 deleted = False
+
+                media_releases = fireEvent('release.for_media', media['_id'], single = True)
+
                 if delete_from == 'all':
+                    # Delete connected releases
+                    for release in media_releases:
+                        db.delete(release)
+
                     db.delete(media)
                     deleted = True
                 else:
 
-                    media_releases = fireEvent('release.for_media', media['_id'], single = True)
-
                     total_releases = len(media_releases)
                     total_deleted = 0
                     new_media_status = None
@@ -379,16 +448,18 @@ class MediaPlugin(MediaBase):
                                 total_deleted += 1
                             new_media_status = 'done'
                         elif delete_from == 'manage':
-                            if release.get('status') == 'done':
+                            if release.get('status') == 'done' or media.get('status') == 'done':
                                 db.delete(release)
                                 total_deleted += 1
 
-                    if (total_releases == total_deleted and media['status'] != 'active') or (delete_from == 'wanted' and media['status'] == 'active'):
+                    if (total_releases == total_deleted) or (total_releases == 0 and not new_media_status) or (not new_media_status and delete_from == 'late'):
                         db.delete(media)
                         deleted = True
                     elif new_media_status:
                         media['status'] = new_media_status
                         db.update(media)
+
+                        fireEvent('media.untag', media['_id'], 'recent', single = True)
                     else:
                         fireEvent('media.restatus', media.get('_id'), single = True)
 
@@ -412,11 +483,16 @@ class MediaPlugin(MediaBase):
     def addSingleDeleteView(self):
 
         for media_type in fireEvent('media.types', merge = True):
-            def tempDelete(*args, **kwargs):
-                return self.deleteView(types = media_type, *args, **kwargs)
-            addApiView('%s.delete' % media_type, tempDelete)
+            tempDelete = lambda *args, **kwargs : self.deleteView(type = media_type, **kwargs)
+            addApiView('%s.delete' % media_type, tempDelete, docs = {
+            'desc': 'Delete a ' + media_type + ' from the wanted list',
+            'params': {
+                'id': {'desc': 'Media ID(s) you want to delete.', 'type': 'int (comma separated)'},
+                'delete_from': {'desc': 'Delete ' + media_type + ' from this page', 'type': 'string: all (default), wanted, manage'},
+            }
+        })
 
-    def restatus(self, media_id):
+    def restatus(self, media_id, tag_recent = True):
 
         try:
             db = get_db()
@@ -428,24 +504,77 @@ class MediaPlugin(MediaBase):
             if not m['profile_id']:
                 m['status'] = 'done'
             else:
-                move_to_wanted = True
+                m['status'] = 'active'
 
-                profile = db.get('id', m['profile_id'])
-                media_releases = fireEvent('release.for_media', m['_id'], single = True)
+                try:
+                    profile = db.get('id', m['profile_id'])
+                    media_releases = fireEvent('release.for_media', m['_id'], single = True)
+                    done_releases = [release for release in media_releases if release.get('status') == 'done']
 
-                for q_identifier in profile['qualities']:
-                    index = profile['qualities'].index(q_identifier)
+                    if done_releases:
 
-                    for release in media_releases:
-                        if q_identifier == release['quality'] and (release.get('status') == 'done' and profile['finish'][index]):
-                            move_to_wanted = False
+                        # Check if we are finished with the media
+                        for release in done_releases:
+                            if fireEvent('quality.isfinish', {'identifier': release['quality'], 'is_3d': release.get('is_3d', False)}, profile, timedelta(seconds = time.time() - release['last_edit']).days, single = True):
+                                m['status'] = 'done'
+                                break
 
-                m['status'] = 'active' if move_to_wanted else 'done'
+                    elif previous_status == 'done':
+                        m['status'] = 'done'
+
+                except RecordNotFound:
+                    log.debug('Failed restatus, keeping previous: %s', traceback.format_exc())
+                    m['status'] = previous_status
 
             # Only update when status has changed
             if previous_status != m['status']:
                 db.update(m)
 
-            return True
+                # Tag media as recent
+                if tag_recent:
+                    self.tag(media_id, 'recent', update_edited = True)
+
+            return m['status']
         except:
             log.error('Failed restatus: %s', traceback.format_exc())
+
+    def tag(self, media_id, tag, update_edited = False):
+
+        try:
+            db = get_db()
+            m = db.get('id', media_id)
+
+            if update_edited:
+                m['last_edit'] = int(time.time())
+
+            tags = m.get('tags') or []
+            if tag not in tags:
+                tags.append(tag)
+                m['tags'] = tags
+                db.update(m)
+
+            return True
+        except:
+            log.error('Failed tagging: %s', traceback.format_exc())
+
+        return False
+
+    def unTag(self, media_id, tag):
+
+        try:
+            db = get_db()
+            m = db.get('id', media_id)
+
+            tags = m.get('tags') or []
+            if tag in tags:
+                new_tags = list(set(tags))
+                new_tags.remove(tag)
+
+                m['tags'] = new_tags
+                db.update(m)
+
+            return True
+        except:
+            log.error('Failed untagging: %s', traceback.format_exc())
+
+        return False

couchpotato/core/media/_base/providers/base.py

@@ -1,4 +1,4 @@
-from urlparse import urlparse
+from six.moves import urllib
 import json
 import re
 import time
@@ -50,7 +50,7 @@ class Provider(Plugin):
         if Env.get('dev'): return True
 
         now = time.time()
-        host = urlparse(test_url).hostname
+        host = urllib.urlparse(test_url).hostname
 
         if self.last_available_check.get(host) < now - 900:
             self.last_available_check[host] = now
@@ -88,10 +88,14 @@ class Provider(Plugin):
 
         if data and len(data) > 0:
             try:
-                data = XMLTree.fromstring(ss(data))
+                data = XMLTree.fromstring(data)
                 return self.getElements(data, item_path)
             except:
-                log.error('Failed to parsing %s: %s', (self.getName(), traceback.format_exc()))
+                try:
+                    data = XMLTree.fromstring(ss(data))
+                    return self.getElements(data, item_path)
+                except:
+                    log.error('Failed to parsing %s: %s', (self.getName(), traceback.format_exc()))
 
         return []
 
@@ -125,6 +129,9 @@ class YarrProvider(Provider):
         else:
             return []
 
+    def buildUrl(self, *args, **kwargs):
+        pass
+
     def login(self):
 
         # Check if we are still logged in every hour
@@ -177,7 +184,7 @@ class YarrProvider(Provider):
         try:
             return self.urlopen(url, headers = {'User-Agent': Env.getIdentifier()}, show_error = False)
         except:
-            log.error('Failed getting nzb from %s: %s', (self.getName(), traceback.format_exc()))
+            log.error('Failed getting release from %s: %s', (self.getName(), traceback.format_exc()))
 
         return 'try_next'
 
@@ -200,7 +207,7 @@ class YarrProvider(Provider):
             self._search(media, quality, results)
         # Search possible titles
         else:
-            media_title = fireEvent('library.query', media, single = True)
+            media_title = fireEvent('library.query', media, include_year = False, single = True)
 
             for title in possibleTitles(media_title):
                 self._searchOnTitle(title, media, quality, results)
@@ -212,7 +219,7 @@ class YarrProvider(Provider):
             if provider and provider == self.getName():
                 return self
 
-            hostname = urlparse(url).hostname
+            hostname = urllib.urlparse(url).hostname
             if host and hostname in host:
                 return self
             else:
@@ -298,7 +305,7 @@ class ResultList(list):
             old_score = new_result['score']
             new_result['score'] = int(old_score * is_correct_weight)
 
-            log.info('Found correct release with weight %.02f, old_score(%d) now scaled to score(%d)', (
+            log.info2('Found correct release with weight %.02f, old_score(%d) now scaled to score(%d)', (
                 is_correct_weight,
                 old_score,
                 new_result['score']

couchpotato/core/media/_base/providers/nzb/binsearch.py

@@ -2,7 +2,7 @@ import re
 import traceback
 
 from bs4 import BeautifulSoup
-from couchpotato.core.helpers.variable import tryInt
+from couchpotato.core.helpers.variable import tryInt, simplifyString
 from couchpotato.core.logger import CPLog
 from couchpotato.core.media._base.providers.nzb.base import NZBProvider
 
@@ -50,8 +50,8 @@ class Base(NZBProvider):
 
                     def extra_check(item):
                         parts = re.search('available:.(?P<parts>\d+)./.(?P<total>\d+)', info.text)
-                        total = tryInt(parts.group('total'))
-                        parts = tryInt(parts.group('parts'))
+                        total = float(tryInt(parts.group('total')))
+                        parts = float(tryInt(parts.group('parts')))
 
                         if (total / parts) < 1 and ((total / parts) < 0.95 or ((total / parts) >= 0.95 and not ('par2' in info.text.lower() or 'pa3' in info.text.lower()))):
                             log.info2('Wrong: \'%s\', not complete: %s out of %s', (item['name'], parts, total))
@@ -65,7 +65,7 @@ class Base(NZBProvider):
 
                     results.append({
                         'id': nzb_id,
-                        'name': title.text,
+                        'name': simplifyString(title.text),
                         'age': tryInt(age),
                         'size': self.parseSize(size_match.group('size')),
                         'url': self.urls['download'] % nzb_id,
@@ -100,6 +100,7 @@ config = [{
             'name': 'binsearch',
             'description': 'Free provider, less accurate. See <a href="https://www.binsearch.info/">BinSearch</a>',
             'wizard': True,
+            'icon': 'iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAAAAAA6mKC9AAAATklEQVQY02NwQAMMWAXOnz+PKvD//3/CAvM//z+fgiwAAs+RBab4PP//vwbFjPlAffgEChzOo2r5fBuIfRAC5w8D+QUofkkp8MHjOWQAAM3Sbogztg2wAAAAAElFTkSuQmCC',
             'options': [
                 {
                     'name': 'enabled',

couchpotato/core/media/_base/providers/nzb/newznab.py

@@ -1,8 +1,7 @@
-from urllib2 import HTTPError
-from urlparse import urlparse
+from six.moves import urllib
 import time
 import traceback
-import urllib2
+import re
 
 from couchpotato.core.helpers.encoding import tryUrlencode, toUnicode
 from couchpotato.core.helpers.rss import RSS
@@ -12,6 +11,7 @@ from couchpotato.core.media._base.providers.base import ResultList
 from couchpotato.core.media._base.providers.nzb.base import NZBProvider
 from couchpotato.environment import Env
 from dateutil.parser import parse
+from requests import HTTPError
 
 
 log = CPLog(__name__)
@@ -20,10 +20,11 @@ log = CPLog(__name__)
 class Base(NZBProvider, RSS):
 
     urls = {
-        'detail': 'details&id=%s',
+        'detail': 'details/%s',
         'download': 't=get&id=%s'
     }
 
+    passwords_regex = 'password|wachtwoord'
     limits_reached = {}
 
     http_time_between_calls = 1  # Seconds
@@ -43,10 +44,8 @@ class Base(NZBProvider, RSS):
 
     def _searchOnHost(self, host, media, quality, results):
 
-        query = self.buildUrl(media, host['api_key'])
-
-        url = '%s&%s' % (self.getUrl(host['host']), query)
-
+        query = self.buildUrl(media, host)
+        url = '%s%s' % (self.getUrl(host['host']), query)
         nzbs = self.getRSSData(url, cache_timeout = 1800, headers = {'User-Agent': Env.getIdentifier()})
 
         for nzb in nzbs:
@@ -79,16 +78,34 @@ class Base(NZBProvider, RSS):
             if spotter:
                 name_extra = spotter
 
+            description = ''
+            if "@spot.net" in nzb_id:
+                try:
+                    # Get details for extended description to retrieve passwords
+                    query = self.buildDetailsUrl(nzb_id, host['api_key'])
+                    url = '%s%s' % (self.getUrl(host['host']), query)
+                    nzb_details = self.getRSSData(url, cache_timeout = 1800, headers = {'User-Agent': Env.getIdentifier()})[0]
+
+                    description = self.getTextElement(nzb_details, 'description')
+
+                    # Extract a password from the description
+                    password = re.search('(?:' + self.passwords_regex + ')(?: *)(?:\:|\=)(?: *)(.*?)\<br\>|\n|$', description, flags = re.I).group(1)
+                    if password:
+                        name += ' {{%s}}' % password.strip()
+                except:
+                    log.debug('Error getting details of "%s": %s', (name, traceback.format_exc()))
+
             results.append({
                 'id': nzb_id,
-                'provider_extra': urlparse(host['host']).hostname or host['host'],
+                'provider_extra': urllib.urlparse(host['host']).hostname or host['host'],
                 'name': toUnicode(name),
                 'name_extra': name_extra,
                 'age': self.calculateAge(int(time.mktime(parse(date).timetuple()))),
                 'size': int(self.getElement(nzb, 'enclosure').attrib['length']) / 1024 / 1024,
                 'url': ((self.getUrl(host['host']) + self.urls['download']) % tryUrlencode(nzb_id)) + self.getApiExt(host),
-                'detail_url': '%sdetails/%s' % (cleanHost(host['host']), tryUrlencode(nzb_id)),
+                'detail_url': (cleanHost(host['host']) + self.urls['detail']) % tryUrlencode(nzb_id),
                 'content': self.getTextElement(nzb, 'description'),
+                'description': description,
                 'score': host['extra_score'],
             })
 
@@ -158,7 +175,7 @@ class Base(NZBProvider, RSS):
         return '&apikey=%s' % host['api_key']
 
     def download(self, url = '', nzb_id = ''):
-        host = urlparse(url).hostname
+        host = urllib.urlparse(url).hostname
 
         if self.limits_reached.get(host):
             # Try again in 3 hours
@@ -166,24 +183,16 @@ class Base(NZBProvider, RSS):
                 return 'try_next'
 
         try:
-            # Get final redirected url
-            log.debug('Checking %s for redirects.', url)
-            req = urllib2.Request(url)
-            req.add_header('User-Agent', self.user_agent)
-            res = urllib2.urlopen(req)
-            finalurl = res.geturl()
-            if finalurl != url:
-                log.debug('Redirect url used: %s', finalurl)
-
-            data = self.urlopen(finalurl, show_error = False)
+            data = self.urlopen(url, show_error = False)
             self.limits_reached[host] = False
             return data
         except HTTPError as e:
-            if e.code == 503:
+            sc = e.response.status_code
+            if sc in [503, 429]:
                 response = e.read().lower()
-                if 'maximum api' in response or 'download limit' in response:
+                if sc == 429 or 'maximum api' in response or 'download limit' in response:
                     if not self.limits_reached.get(host):
-                        log.error('Limit reached for newznab provider: %s', host)
+                        log.error('Limit reached / to many requests for newznab provider: %s', host)
                     self.limits_reached[host] = time.time()
                     return 'try_next'
 
@@ -191,6 +200,15 @@ class Base(NZBProvider, RSS):
 
         return 'try_next'
 
+    def buildDetailsUrl(self, nzb_id, api_key):
+        query = tryUrlencode({
+            't': 'details',
+            'id': nzb_id,
+            'apikey': api_key,
+        })
+        return query
+
+
 
 config = [{
     'name': 'newznab',
@@ -203,8 +221,9 @@ config = [{
             'description': 'Enable <a href="http://newznab.com/" target="_blank">NewzNab</a> such as <a href="https://nzb.su" target="_blank">NZB.su</a>, \
                 <a href="https://nzbs.org" target="_blank">NZBs.org</a>, <a href="http://dognzb.cr/" target="_blank">DOGnzb.cr</a>, \
                 <a href="https://github.com/spotweb/spotweb" target="_blank">Spotweb</a>, <a href="https://nzbgeek.info/" target="_blank">NZBGeek</a>, \
-                <a href="https://smackdownonyou.com" target="_blank">SmackDown</a>, <a href="https://www.nzbfinder.ws" target="_blank">NZBFinder</a>',
+                <a href="https://www.nzbfinder.ws" target="_blank">NZBFinder</a>',
             'wizard': True,
+            'icon': 'iVBORw0KGgoAAAANSUhEUgAAABAAAAAQAgMAAABinRfyAAAACVBMVEVjhwD///86aRovd/sBAAAAMklEQVQI12NgAIPQUCCRmQkjssDEShiRuRIqwZqZGcDAGBrqANUhGgIkWAOABKMDxCAA24UK50b26SAAAAAASUVORK5CYII=',
             'options': [
                 {
                     'name': 'enabled',
@@ -213,30 +232,30 @@ config = [{
                 },
                 {
                     'name': 'use',
-                    'default': '0,0,0,0,0,0'
+                    'default': '0,0,0,0,0'
                 },
                 {
                     'name': 'host',
-                    'default': 'api.nzb.su,dognzb.cr,nzbs.org,https://index.nzbgeek.info, https://smackdownonyou.com, https://www.nzbfinder.ws',
+                    'default': 'api.nzb.su,api.dognzb.cr,nzbs.org,https://api.nzbgeek.info,https://www.nzbfinder.ws',
                     'description': 'The hostname of your newznab provider',
                 },
                 {
                     'name': 'extra_score',
                     'advanced': True,
                     'label': 'Extra Score',
-                    'default': '0,0,0,0,0,0',
+                    'default': '0,0,0,0,0',
                     'description': 'Starting score for each release found via this provider.',
                 },
                 {
                     'name': 'custom_tag',
                     'advanced': True,
                     'label': 'Custom tag',
-                    'default': ',,,,,',
+                    'default': ',,,,',
                     'description': 'Add custom tags, for example add rls=1 to get only scene releases from nzbs.org',
                 },
                 {
                     'name': 'api_key',
-                    'default': ',,,,,',
+                    'default': ',,,,',
                     'label': 'Api Key',
                     'description': 'Can be found on your profile page',
                     'type': 'combined',

couchpotato/core/media/_base/providers/nzb/nzbclub.py

@@ -80,6 +80,7 @@ config = [{
             'name': 'NZBClub',
             'description': 'Free provider, less accurate. See <a href="https://www.nzbclub.com/">NZBClub</a>',
             'wizard': True,
+            'icon': 'iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAACEUlEQVQ4y3VSMWgUQRR9/8/s7OzeJSdnTsVGghLEYBNQjBpQiRBFhIB2EcHG1kbs0murhZAmVocExEZQ0c7CxkLINYcJJpoYj9wZcnu72fF21uJSXMzuhyne58/j/fcf4b+KokgBIOSU53lxP5b9oNVqDT36dH+5UjoiKvIwPFEEgWBshGZ3E7/NOupL9fMjx0e+ZhKsrq+c/FPZKJi0w4FsQXMBDEJsd7BNW9h2tuyP9vfTALIJkMIu1hYRtINM+dpzcWc0sbkreK4fUEogyraAmKGF3+7vcT/wtR9QwkCabSAzQQuvk0uglAo5YaQ5DASGYjfMXcHVOqKu6NmR7iehlKAdHWUqWPv1c3i+9uwVdRlEBGaGEAJCCrDo9ShhvF6qPq8tL57bp+DbRn2sHtUuCY9YphLMu5921VhrwYJ5tbt0tt6sjQP4vEfB2Ikz7/ytwbeR6ljHkXCUA6UcOLtPOg4MYhtH8ZcLw5er+xQMDAwEURRNl96X596Y6oxFwsw9fmtTOAr2Ik19nL365FZpsLSdnQPPM8aYewc+lDcX4rkHqbQMAGTJXulOLzycmr1bKBTi3DOGYagajcahiaOT89fbM0/dxEsUu3aidfPljWO3HzebzYNBELi5Z5RSJlrrHd/3w8lT114MrVTWOn875fHRiYVisRhorWMpZXdvNnLKGCOstb0AMlulVJI19w/+nceU4D0aCwAAAABJRU5ErkJggg==',
             'options': [
                 {
                     'name': 'enabled',

couchpotato/core/media/_base/providers/nzb/nzbindex.py

@@ -1,125 +0,0 @@
-import re
-import time
-
-from bs4 import BeautifulSoup
-from couchpotato.core.helpers.encoding import toUnicode
-from couchpotato.core.helpers.rss import RSS
-from couchpotato.core.helpers.variable import tryInt
-from couchpotato.core.logger import CPLog
-from couchpotato.core.event import fireEvent
-from couchpotato.core.media._base.providers.nzb.base import NZBProvider
-from dateutil.parser import parse
-
-
-log = CPLog(__name__)
-
-
-class Base(NZBProvider, RSS):
-
-    urls = {
-        'download': 'https://www.nzbindex.com/download/',
-        'search': 'https://www.nzbindex.com/rss/?%s',
-    }
-
-    http_time_between_calls = 1  # Seconds
-
-    def _search(self, media, quality, results):
-
-        nzbs = self.getRSSData(self.urls['search'] % self.buildUrl(media, quality))
-
-        for nzb in nzbs:
-
-            enclosure = self.getElement(nzb, 'enclosure').attrib
-            nzbindex_id = int(self.getTextElement(nzb, "link").split('/')[4])
-
-            title = self.getTextElement(nzb, "title")
-
-            match = fireEvent('matcher.parse', title, parser='usenet', single = True)
-            if not match.chains:
-                log.info('Unable to parse release with title "%s"', title)
-                continue
-
-            # TODO should we consider other lower-weight chains here?
-            info = fireEvent('matcher.flatten_info', match.chains[0].info, single = True)
-
-            release_name = fireEvent('matcher.construct_from_raw', info.get('release_name'), single = True)
-
-            file_name = info.get('detail', {}).get('file_name')
-            file_name = file_name[0] if file_name else None
-
-            title = release_name or file_name
-
-            # Strip extension from parsed title (if one exists)
-            ext_pos = title.rfind('.')
-
-            # Assume extension if smaller than 4 characters
-            # TODO this should probably be done a better way
-            if len(title[ext_pos + 1:]) <= 4:
-                title = title[:ext_pos]
-
-            if not title:
-                log.info('Unable to find release name from match')
-                continue
-
-            try:
-                description = self.getTextElement(nzb, "description")
-            except:
-                description = ''
-
-            def extra_check(item):
-                if '#c20000' in item['description'].lower():
-                    log.info('Wrong: Seems to be passworded: %s', item['name'])
-                    return False
-
-                return True
-
-            results.append({
-                'id': nzbindex_id,
-                'name': title,
-                'age': self.calculateAge(int(time.mktime(parse(self.getTextElement(nzb, "pubDate")).timetuple()))),
-                'size': tryInt(enclosure['length']) / 1024 / 1024,
-                'url': enclosure['url'],
-                'detail_url': enclosure['url'].replace('/download/', '/release/'),
-                'description': description,
-                'get_more_info': self.getMoreInfo,
-                'extra_check': extra_check,
-            })
-
-    def getMoreInfo(self, item):
-        try:
-            if '/nfo/' in item['description'].lower():
-                nfo_url = re.search('href=\"(?P<nfo>.+)\" ', item['description']).group('nfo')
-                full_description = self.getCache('nzbindex.%s' % item['id'], url = nfo_url, cache_timeout = 25920000)
-                html = BeautifulSoup(full_description)
-                item['description'] = toUnicode(html.find('pre', attrs = {'id': 'nfo0'}).text)
-        except:
-            pass
-
-
-config = [{
-    'name': 'nzbindex',
-    'groups': [
-        {
-            'tab': 'searcher',
-            'list': 'nzb_providers',
-            'name': 'nzbindex',
-            'description': 'Free provider, less accurate. See <a href="https://www.nzbindex.com/">NZBIndex</a>',
-            'wizard': True,
-            'options': [
-                {
-                    'name': 'enabled',
-                    'type': 'enabler',
-                    'default': True,
-                },
-                {
-                    'name': 'extra_score',
-                    'advanced': True,
-                    'label': 'Extra Score',
-                    'type': 'int',
-                    'default': 0,
-                    'description': 'Starting score for each release found via this provider.',
-                }
-            ],
-        },
-    ],
-}]

couchpotato/core/media/_base/providers/nzb/omgwtfnzbs.py

@@ -1,4 +1,4 @@
-from urlparse import urlparse, parse_qs
+from six.moves import urllib
 import time
 
 from couchpotato.core.event import fireEvent
@@ -52,7 +52,7 @@ class Base(NZBProvider, RSS):
         for nzb in nzbs:
 
             enclosure = self.getElement(nzb, 'enclosure').attrib
-            nzb_id = parse_qs(urlparse(self.getTextElement(nzb, 'link')).query).get('id')[0]
+            nzb_id = urllib.parse_qs(urllib.urlparse(self.getTextElement(nzb, 'link')).query).get('id')[0]
 
             results.append({
                 'id': nzb_id,
@@ -74,6 +74,7 @@ config = [{
             'name': 'OMGWTFNZBs',
             'description': 'See <a href="http://omgwtfnzbs.org/">OMGWTFNZBs</a>',
             'wizard': True,
+            'icon': 'iVBORw0KGgoAAAANSUhEUgAAABAAAAAQEAIAAADAAbR1AAADbElEQVR4AZ2UW0ybZRiAy/OvdHaLYvB0YTRIFi7GkM44zRLmIfNixkWdiRMyYoxRE8/TC7MYvXCGEBmr3mxLwVMwY0wYA7e6Wso4lB6h/U9taSlMGIfBXLYlJMyo0S///2dJI5lxN8/F2/f9nu9737e/jYmXr6KTbN9BGG9HE/NotQ76UWziNzrXFiETk/5ARUNH+7+0kW7fSgTl0VKGOLZzidOkmuuIo7q2oTArNLPIzhdIkqXkerFOm2CaD/5bcKrjIL2c3fkhPxOq93Kcb91v46fV9TQKF4TgV/TbUsQtzfCaK6jMOd5DJrguSIIhexmqqVxN0FXbRR8/ND/LYTTj6J7nl2gnL47OkDW4KJhnQHCa6JpKVNJGA3OC58nwBJoZ//ebbIyKpBxjrr0o1q1FMRkrKXZnHWF85VvxMrJxibwhGyd0f5bLnKzqJs1k0Sfo+EU8hdAUvkbcwKEgs2D0OiV4jmmD1zb+Tp6er0JMMvDxPo5xev9zTBF683NS+N56n1YiB95B5crr93KRuKhKI0tb0Kw2mgLLqTjLEWO8424i9IvURaYeOckwf3+/yCC9e3bQQ/MuD+Monk0k+XFXMUfx7z5EEP+XlXi5tLlMxH8zLppw7idJrugcus30kC86gc7UrQqjLIukM8zWHOACeU+TiMxXN6ExVOkgz4lvPEzice1GIVhxhG4CrZvpl6TH55giKWqXGLy9hZh5aUtgDSew/msSyCKpl+DDNfxJc8NBIsxUxUnz14O/oONu+IIIvso9TLBQ1SY5rUhuSzUhAqJ2mRXBLDOCeUtgUZXsaObT8BffhUJPqWgiV+3zKKzYH0ClvTRLhD77HIqVkyh5jThnivehoG+qJctIRSPn6bxvO4FCgTl9c1DmbpjLajbQFE8aW5SU3rg+zOPGUjTUF9NFpLEbH2c/KmGYlY69/GQJVtGMSUcEp9eCbB1nctbxHTLRdTUkGDf+B02uGWRG3OvpJ/zSMwzif+oxVBID3cQKBavLCiPmB2PM2UuSCUPgrX4VDb97AwEG67bh4+KTOlncvu3M31BwA5rLHbCfEjwkNDky9e/SSbSxnD46Pg0RJtpXRvhmBSZHpRjWtKwFybjuQeXaKxto4WjLZZZvVmC17pZLJFkwxm5++PS2Mrwc7nyIMYZe/IzoP5d6QgEybqTXAAAAAElFTkSuQmCC',
             'options': [
                 {
                     'name': 'enabled',

couchpotato/core/media/_base/providers/torrent/awesomehd.py

@@ -61,7 +61,7 @@ class Base(TorrentProvider):
                         'name': re.sub('[^A-Za-z0-9\-_ \(\).]+', '', '%s (%s) %s' % (name, year, torrent_desc)),
                         'url': self.urls['download'] % (torrent_id, authkey, self.conf('passkey')),
                         'detail_url': self.urls['detail'] % torrent_id,
-                        'size': self.parseSize(entry.find('size').get_text()),
+                        'size': tryInt(entry.find('size').get_text()) / 1048576,
                         'seeders': tryInt(entry.find('seeders').get_text()),
                         'leechers': tryInt(entry.find('leechers').get_text()),
                         'score': torrentscore
@@ -78,8 +78,9 @@ config = [{
             'tab': 'searcher',
             'list': 'torrent_providers',
             'name': 'Awesome-HD',
-            'description': 'See <a href="https://awesome-hd.net">AHD</a>',
+            'description': '<a href="https://awesome-hd.net">AHD</a>',
             'wizard': True,
+            'icon': 'iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAC+UlEQVR4AV1SO0y6dxQ9H4g8CoIoohZ5NA0aR2UgkYpNB5uocTSaLlrDblMH09Gt8d90r3YpJkanxjA4GGkbO7RNxSABq8jDGnkpD+UD5NV7Bxvbk9wvv+/3uPece66A/yEWi42FQqHVfD7/cbPZtIEglUpjOp3uZHR0dBvAn3gDIRqNgjE4OKj0+Xzf3NzcfD4wMCCjf5TLZbTbbajVatzf3+Pu7q5uNpt35ufnvwBQAScQRREEldfr9RWLxan+/n5YrVa+jFarhVfQQyQSCU4EhULhX15engEgSrjC0dHRVqlUmjQYDBgaGgKtuTqz4mTgIoVCASaTCX19fajVapOHh4dbFJBks9mxcDi8qtFoJEajkfVyJWi1WkxMTMDhcIAT8x6D7/Dd6+vr1fHx8TGp2+3+iqo5+YCzBwIBToK5ubl/mQwPDyMSibAs2Gw2UHNRrValz8/PDUk8Hv9EqVRCr9fj4uICTNflcqFer+Pg4AB7e3uoVCq8x9Rxfn6O7u5uqFQq8FspZXxHTekggByA3W4Hr9PpNDeRL3I1cMhkMrBrnZ2dyGQyvNYIs7OzVbJNPjIyAraLwYdcjR8wXl5eIJfLwRIFQQDLYkm3t7c1CdGPPT4+cpOImp4PODMeaK+n10As2jBbrHifHOjS6qAguVFimkqlwAMmIQnHV1dX4NDQhVwuhyZTV6pgIktzDzkkk0lEwhEEzs7ASQr5Ai4vL1nuccfCwsLO/v6+p9FoyJhF6ekJro/cPCzIZLNQa7rQoK77/SdgWWpKkCaJ5EB9aWnpe6nH40nRMBnJV4f5gw+FX3/5GX/8/htXRZdOzzqhJWn6nl6YbTZqqhrhULD16fT0d8FgcFtYW1vD5uamfGVl5cd4IjldKhZACdkJvKfWUANrxEaJV4hiGVaL1b+7653hXzwRZQr2X76xsfG1xWIRaZzbNPv/CdrjEL9cX/+WXFBSgEPgzxuwG3Yans9OT0+naBZMIJDNfzudzp8WFxd/APAX3uAf9WOTxOPLdosAAAAASUVORK5CYII=',
             'options': [
                 {
                     'name': 'enabled',

couchpotato/core/media/_base/providers/torrent/base.py

@@ -44,7 +44,8 @@ class TorrentProvider(YarrProvider):
 
                 prop_name = 'proxy.%s' % proxy
                 last_check = float(Env.prop(prop_name, default = 0))
-                if last_check > time.time() - 1209600:
+
+                if last_check > time.time() - 86400:
                     continue
 
                 data = ''

couchpotato/core/media/_base/providers/torrent/bithdtv.py

@@ -25,7 +25,7 @@ class Base(TorrentProvider):
 
     def _search(self, media, quality, results):
 
-        query = self.buildUrl(media)
+        query = self.buildUrl(media, quality)
 
         url = "%s&%s" % (self.urls['search'], query)
 
@@ -93,8 +93,9 @@ config = [{
             'tab': 'searcher',
             'list': 'torrent_providers',
             'name': 'BiT-HDTV',
-            'description': 'See <a href="http://bit-hdtv.com">BiT-HDTV</a>',
+            'description': '<a href="http://bit-hdtv.com">BiT-HDTV</a>',
             'wizard': True,
+            'icon': 'iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAIAAACQkWg2AAAABnRSTlMAAAAAAABupgeRAAABMklEQVR4AZ3Qu0ojcQCF8W9MJcQbJNgEEQUbQVIqWgnaWfkIvoCgggixEAmIhRtY2GV3w7KwU61B0EYIxmiw0YCik84ipaCuc0nmP5dcjIUgOjqDvxf4OAdf9mnMLcUJyPyGSCP+YRdC+Kp8iagJKhuS+InYRhTGgDbeV2uEMand4ZRxizjXHQEimxhraAnUr73BNqQxMiNeV2SwcjTLEVtb4Zl10mXutvOWm2otw5Sxz6TGTbdd6ncuYvVLXAXrvM+ruyBpy1S3JLGDfUQ1O6jn5vTsrJXvqSt4UNfj6vxTRPxBHER5QeSirhLGk/5rWN+ffB1XZuxjnDy1q87m7TS+xOGA+Iv4gfkbaw+nOMXHDHnITGEk0VfRFnn4Po4vNYm6RGukmggR0L08+l+e4HMeASo/i6AJUjLgAAAAAElFTkSuQmCC',
             'options': [
                 {
                     'name': 'enabled',

couchpotato/core/media/_base/providers/torrent/bitsoup.py

@@ -1,7 +1,6 @@
 import traceback
 
-from bs4 import BeautifulSoup
-from couchpotato.core.helpers.encoding import simplifyString, tryUrlencode
+from bs4 import BeautifulSoup, SoupStrainer
 from couchpotato.core.helpers.variable import tryInt
 from couchpotato.core.logger import CPLog
 from couchpotato.core.media._base.providers.torrent.base import TorrentProvider
@@ -16,25 +15,23 @@ class Base(TorrentProvider):
         'test': 'https://www.bitsoup.me/',
         'login': 'https://www.bitsoup.me/takelogin.php',
         'login_check': 'https://www.bitsoup.me/my.php',
-        'search': 'https://www.bitsoup.me/browse.php?',
+        'search': 'https://www.bitsoup.me/browse.php?%s',
         'baseurl': 'https://www.bitsoup.me/%s',
     }
 
     http_time_between_calls = 1  # Seconds
+    only_tables_tags = SoupStrainer('table')
+
+    torrent_name_cell = 1
+    torrent_download_cell = 2
 
     def _searchOnTitle(self, title, movie, quality, results):
 
-        q = '"%s" %s' % (simplifyString(title), movie['info']['year'])
-        arguments = tryUrlencode({
-            'search': q,
-        })
-        url = "%s&%s" % (self.urls['search'], arguments)
-
-        url = self.urls['search'] % self.buildUrl(movie, quality)
+        url = self.urls['search'] % self.buildUrl(title, movie, quality)
         data = self.getHTMLData(url)
 
         if data:
-            html = BeautifulSoup(data)
+            html = BeautifulSoup(data, 'html.parser', parse_only = self.only_tables_tags)
 
             try:
                 result_table = html.find('table', attrs = {'class': 'koptekst'})
@@ -46,8 +43,8 @@ class Base(TorrentProvider):
 
                     all_cells = result.find_all('td')
 
-                    torrent = all_cells[1].find('a')
-                    download = all_cells[3].find('a')
+                    torrent = all_cells[self.torrent_name_cell].find('a')
+                    download = all_cells[self.torrent_download_cell].find('a')
 
                     torrent_id = torrent['href']
                     torrent_id = torrent_id.replace('details.php?id=', '')
@@ -55,9 +52,9 @@ class Base(TorrentProvider):
 
                     torrent_name = torrent.getText()
 
-                    torrent_size = self.parseSize(all_cells[7].getText())
-                    torrent_seeders = tryInt(all_cells[9].getText())
-                    torrent_leechers = tryInt(all_cells[10].getText())
+                    torrent_size = self.parseSize(all_cells[8].getText())
+                    torrent_seeders = tryInt(all_cells[10].getText())
+                    torrent_leechers = tryInt(all_cells[11].getText())
                     torrent_url = self.urls['baseurl'] % download['href']
                     torrent_detail_url = self.urls['baseurl'] % torrent['href']
 
@@ -94,8 +91,9 @@ config = [{
             'tab': 'searcher',
             'list': 'torrent_providers',
             'name': 'Bitsoup',
-            'description': 'See <a href="https://bitsoup.me">Bitsoup</a>',
+            'description': '<a href="https://bitsoup.me">Bitsoup</a>',
             'wizard': True,
+            'icon': 'iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAIAAACQkWg2AAAB8ElEQVR4AbWSS2sTURiGz3euk0mswaE37HhNhIrajQheFgF3rgR/lAt/gOBCXNZlo6AbqfUWRVCxi04wqUnTRibpJLaJzdzOOZ6WUumyC5/VHOb9eN/FA91uFx0FjI4IPfgiGLTWH73tn348GKmN7ijD0d2b41fO5qJEaX24AWNIUrVQCTTJ3Llx6vbV6Vtzk7Gi9+ebi996guFDDYAQAVj4FExP5qdOZB49W62t/zH3hECcwsPnbWeMXz6Xi2K1f0ApeK3hMCHHbP5gvvoriBgFAAQJEAxhjJ4u+YWTNsVI6b1JgtPWZkoIefKy4fcii2OTw2BABs7wj3bYDlLL4rvjGWOdTser1j5Xf7c3Q/MbHQYApxItvnm31mhQQ71eX2vUB76/vsWB2hg0QuogrMwLIG8P3InM2/eVGXeDViqVwWB79vRU2lgJYmdHcgXCTAXQFJTN5HguvDCR2Hxsxe8EvT54nlcul5vNpqDIEgwRQanAhAAABgRIyiQcjpIkkTOuWyqVoN/vSylX67XXH74uV1vHRUyxxFqbLBCSmBpiXSq6xcL5QrGYzWZ3XQIAwdlOJB+/aL764ucdmncYs0WsCI7kvTnn+qyDMEnTVCn1Tz5KsBFg6fvWcmsUAcnYNC/g2hnromvvqbHvxv+39S+MX+bWkFXwAgAAAABJRU5ErkJggg==',
             'options': [
                 {
                     'name': 'enabled',

couchpotato/core/media/_base/providers/torrent/hdbits.py

@@ -71,7 +71,9 @@ config = [{
             'tab': 'searcher',
             'list': 'torrent_providers',
             'name': 'HDBits',
-            'description': 'See <a href="http://hdbits.org">HDBits</a>',
+            'wizard': True,
+            'description': '<a href="http://hdbits.org">HDBits</a>',
+            'icon': 'iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAIAAACQkWg2AAABi0lEQVR4AZWSzUsbQRjGdyabTcvSNPTSHlpQQeMHJApC8CJRvHgQQU969+LJP8G7f4N3DwpeFRQvRr0EKaUl0ATSpkigUNFsMl/r9NmZLCEHA/nNO5PfvMPDm0DI6fV3ZxiolEICe1oZCBVCCmBPKwOh2ErKBHGE4KYEXBpSLkUlqO4LcM7f+6nVhRnOhSkOz/hexk+tL+YL0yPF2YmN4tynD++4gTLGkNNac9YFLoREBR1+cnF3dFY6v/m6PD+FaXiNJtgA4xYbABxiGrz6+6HWaI5/+Qh37YS0/3Znc8UxwNGBIIBX22z+/ZdJ+4wzyjpR4PEpODg8tgUXBv2iWUzSpa12B0IR6n6lvt8Aek2lZHb084+fdRNgrwY8z81PjhVy2d2ttUrtV/lbBa+JXGEpDMPnoF2tN1QYRqVUtf6nFbThb7wk7le395elcqhASLb39okDiHY00VCtCTEHwSiH4AI0lkOiT1dwMeSfT3SRxiQWNO7Zwj1egkoVIQFMKvSiC3bcjXq9Jf8DcDIRT3hh10kAAAAASUVORK5CYII=',
             'options': [
                 {
                     'name': 'enabled',

couchpotato/core/media/_base/providers/torrent/ilovetorrents.py

@@ -3,7 +3,7 @@ import traceback
 
 from bs4 import BeautifulSoup
 from couchpotato.core.helpers.encoding import toUnicode, tryUrlencode
-from couchpotato.core.helpers.variable import tryInt
+from couchpotato.core.helpers.variable import tryInt, splitString
 from couchpotato.core.logger import CPLog
 from couchpotato.core.media._base.providers.torrent.base import TorrentProvider
 
@@ -15,7 +15,7 @@ class Base(TorrentProvider):
 
     urls = {
         'download': 'https://www.ilovetorrents.me/%s',
-        'detail': 'https//www.ilovetorrents.me/%s',
+        'detail': 'https://www.ilovetorrents.me/%s',
         'search': 'https://www.ilovetorrents.me/browse.php?search=%s&page=%s&cat=%s',
         'test': 'https://www.ilovetorrents.me/',
         'login': 'https://www.ilovetorrents.me/takelogin.php',
@@ -47,17 +47,24 @@ class Base(TorrentProvider):
             data = self.getHTMLData(search_url)
             if data:
                 try:
-                    soup = BeautifulSoup(data)
 
-                    results_table = soup.find('table', attrs = {'class': 'koptekst'})
+                    results_table = None
+
+                    data_split = splitString(data, '<table')
+                    soup = None
+                    for x in data_split:
+                        soup = BeautifulSoup(x)
+                        results_table = soup.find('table', attrs = {'class': 'koptekst'})
+                        if results_table:
+                            break
+
                     if not results_table:
                         return
 
                     try:
                         pagelinks = soup.findAll(href = re.compile('page'))
-                        pageNumbers = [int(re.search('page=(?P<pageNumber>.+'')', i['href']).group('pageNumber')) for i in pagelinks]
-                        total_pages = max(pageNumbers)
-
+                        page_numbers = [int(re.search('page=(?P<page_number>.+'')', i['href']).group('page_number')) for i in pagelinks]
+                        total_pages = max(page_numbers)
                     except:
                         pass
 
@@ -139,6 +146,7 @@ config = [{
             'name': 'ILoveTorrents',
             'description': 'Where the Love of Torrents is Born',
             'wizard': True,
+            'icon': 'iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAIAAACQkWg2AAACPUlEQVR4AYWM0U9SbxjH3+v266I/oNvWZTfd2J1d0ZqbZEFwWrUImOKs4YwtumFKZvvlJJADR2TCQQlMPKg5NmpREgaekAPnBATKgmK1LqQlx6awHnZWF1Tr2Xfvvs+7z+dB0mlO7StpAh+M4S/2jbo3w8+xvJvlnSneEt+10zwer5ujNUOoChjALWFw5XOwdCAk/P57cGvPl+Oht0W7VJHN5NC1uW1BON4hGjXbwpVWMZhsy9v7sEIXAsDNYBXgdkEoIKyWD2CF8ut/aOXTZc/fBSgLWw1BgA4BDHOV0GkT90cBQpXahU5TFomsb38XhJC5/Tbh1P8c6rJlBeGfAeyMhUFwNVcs9lxV9Ot0dwmyd+mrNvRtbJ2fSPC6Z3Vsvub2z3sDFACAAYzk0+kUyxEkyfN7PopqNBro55A+P6yPKIrL5zF1HwjdeBJJCObIsZO79bo3sHhWhglo5WMV3mazuVPb4fLvSL8/FAkB1hK6rXQPwYhMyROK8VK5LAiH/jsMt0HQjxiN4/ePdoilllcqDyt3Mkg8mRBNbIhMb8RERkowQA/p76g0/UDDdCoNmDminM0qSK5vlpE5kugCHhNPxntwWmJPYTMZtYcFR6ABHQsVRlYLukVORaaULvqKI46keFSCv77kSPS6kxrPptLNDHgz16fWBtyxe6v5h08LUy+KI8ushqTPWWIX8Sg6b45IrGtyW6zXFb/hpQf9m3oqfWuB0fpSw0uZ4WB69En69uOk2rmO2V52PXj+A/mI4ESKpb2HAAAAAElFTkSuQmCC',
             'options': [
                 {
                     'name': 'enabled',

couchpotato/core/media/_base/providers/torrent/iptorrents.py

@@ -120,8 +120,9 @@ config = [{
             'tab': 'searcher',
             'list': 'torrent_providers',
             'name': 'IPTorrents',
-            'description': 'See <a href="http://www.iptorrents.com">IPTorrents</a>',
+            'description': '<a href="http://www.iptorrents.com">IPTorrents</a>',
             'wizard': True,
+            'icon': 'iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAABRklEQVR42qWQO0vDUBiG8zeKY3EqQUtNO7g0J6ZJ1+ifKIIFQXAqDYKCyaaYxM3udrZLHdRFhXrZ6liCW6mubfk874EESgqaeOCF7/Y8hEh41aq6yZi2nyZgBGya9XKtZs4No05pAkZV2YbEmyMMsoSxLQeC46wCTdPPY4HruPQyGIhF97qLWsS78Miydn4XdK46NJ9OsQAYBzMIMf8MQ9wtCnTdWCaIDx/u7uljOIQEe0hiIWPamSTLay3+RxOCSPI9+RJAo7Er9r2bnqjBFAqyK+VyK4f5/Cr5ni8OFKVCz49PFI5GdNvvU7ttE1M1zMU+8AMqFksEhrMnQsBDzqmDAwzx2ehRLwT7yyCI+vSC99c3mozH1NxrJgWWtR1BOECfEJSVCm6WCzJGCA7+IWhBsM4zywDPwEp4vCjx2DzBH2ODAfsDb33Ps6dQwJgAAAAASUVORK5CYII=',
             'options': [
                 {
                     'name': 'enabled',

couchpotato/core/media/_base/providers/torrent/kickasstorrents.py

@@ -32,8 +32,11 @@ class Base(TorrentMagnetProvider):
     proxy_list = [
         'https://kickass.to',
         'http://kickass.pw',
-        'http://www.kickassunblock.info',
-        'http://www.kickassproxy.info',
+        'http://kickassto.come.in',
+        'http://katproxy.ws',
+        'http://kickass.bitproxy.eu',
+        'http://katph.eu',
+        'http://kickassto.come.in',
     ]
 
     def _search(self, media, quality, results):
@@ -65,12 +68,13 @@ class Base(TorrentMagnetProvider):
                                 if column_name:
 
                                     if column_name == 'name':
-                                        link = td.find('div', {'class': 'torrentname'}).find_all('a')[1]
-                                        new['id'] = temp.get('id')[-8:]
+                                        link = td.find('div', {'class': 'torrentname'}).find_all('a')[2]
+                                        new['id'] = temp.get('id')[-7:]
                                         new['name'] = link.text
                                         new['url'] = td.find('a', 'imagnet')['href']
                                         new['detail_url'] = self.urls['detail'] % (self.getDomain(), link['href'][1:])
-                                        new['score'] = 20 if td.find('a', 'iverif') else 0
+                                        new['verified'] = True if td.find('a', 'iverify') else False
+                                        new['score'] = 100 if new['verified'] else 0
                                     elif column_name is 'size':
                                         new['size'] = self.parseSize(td.text)
                                     elif column_name is 'age':
@@ -82,6 +86,10 @@ class Base(TorrentMagnetProvider):
 
                                 nr += 1
 
+                            # Only store verified torrents
+                            if self.conf('only_verified') and not new['verified']:
+                                continue
+
                             results.append(new)
                     except:
                         log.error('Failed parsing KickAssTorrents: %s', traceback.format_exc())
@@ -123,8 +131,9 @@ config = [{
             'tab': 'searcher',
             'list': 'torrent_providers',
             'name': 'KickAssTorrents',
-            'description': 'See <a href="https://kat.ph/">KickAssTorrents</a>',
+            'description': '<a href="https://kat.ph/">KickAssTorrents</a>',
             'wizard': True,
+            'icon': 'iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAACD0lEQVR42pXK20uTcRjA8d/fsJsuap0orBuFlm3hir3JJvQOVmuwllN20Lb2isI2nVHKjBqrCWYaNnNuBrkSWxglhDVJOkBdSWUOq5FgoiOrMdRJ2xPPxW+8OUf1ge/FcyCUSVe2qedK5U/OxNTTXRNXEQ52Glb4O6dNEfK1auJkvRY7+/zxnQbA/D596laXcY3OWOiaIX2393SGznUmxkUo/YkDgqHemuzobQ7+NV+reo5Q1mqp68GABdY3+/EloO+JeN4tEqiFU8f3CwhyWo9E7wfMgI0ELTDx0AvjIxcgvZoC9P7NMN7yMmrFeoKa68rfDfmrARsNN0Ihr55cx59ctZWSiwS5bLKpwW4dYJH+M/B6/CYszE0BFZ+egG+Ln+HRoBN/cpl1pV6COIMkOnBVA/w+fXgGKJVM4LxhumMleoL06hJ3wKcCfl+/TAKKx17gnFePRwkqxR4BQSpFkbCrrQJueI7mWpyfATQ9OQY43+uv/+PutBycJ3y2qn2x7jY50GJvnwLKZjOwspyE5I8F4N+1yr1uwqcs3ym63Hwo29EiAyzUWQVr6WVAS4lZCPutQG/2GtES2YiW3d3XflYKtL72kzAcdEDHeSa3czeIMyyz/TApRKvcFfE0isHbJMnrHCf6xTLb1ORvWNlWo91cvHrJUQo0o6ZoRi7dIiT/g2WEDi27Iyov21xMCvgNfXvtwIACfHwAAAAASUVORK5CYII=',
             'options': [
                 {
                     'name': 'enabled',
@@ -151,6 +160,13 @@ config = [{
                     'default': 40,
                     'description': 'Will not be (re)moved until this seed time (in hours) is met.',
                 },
+                {
+                    'name': 'only_verified',
+                    'advanced': True,
+                    'type': 'bool',
+                    'default': False,
+                    'description': 'Only search for verified releases.'
+                },
                 {
                     'name': 'extra_score',
                     'advanced': True,

couchpotato/core/media/_base/providers/torrent/passthepopcorn.py

@@ -1,4 +1,4 @@
-import htmlentitydefs
+from six.moves import html_entities
 import json
 import re
 import time
@@ -64,6 +64,10 @@ class Base(TorrentProvider):
                         torrentdesc += ' HQ'
                         if self.conf('prefer_golden'):
                             torrentscore += 5000
+                    if 'FreeleechType' in torrent:
+                        torrentdesc += ' Freeleech'
+                        if self.conf('prefer_freeleech'):
+                            torrentscore += 7000
                     if 'Scene' in torrent and torrent['Scene']:
                         torrentdesc += ' Scene'
                         if self.conf('prefer_scene'):
@@ -141,15 +145,15 @@ class Base(TorrentProvider):
                 # character reference
                 try:
                     if txt[:3] == "&#x":
-                        return unichr(int(txt[3:-1], 16))
+                        return six.unichr(int(txt[3:-1], 16))
                     else:
-                        return unichr(int(txt[2:-1]))
+                        return six.unichr(int(txt[2:-1]))
                 except ValueError:
                     pass
             else:
                 # named entity
                 try:
-                    txt = unichr(htmlentitydefs.name2codepoint[txt[1:-1]])
+                    txt = six.unichr(html_entities.name2codepoint[txt[1:-1]])
                 except KeyError:
                     pass
             return txt  # leave as is
@@ -187,8 +191,9 @@ config = [{
             'tab': 'searcher',
             'list': 'torrent_providers',
             'name': 'PassThePopcorn',
-            'description': 'See <a href="https://passthepopcorn.me">PassThePopcorn.me</a>',
+            'description': '<a href="https://passthepopcorn.me">PassThePopcorn.me</a>',
             'wizard': True,
+            'icon': 'iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAIAAACQkWg2AAAARklEQVQoz2NgIAP8BwMiGWRpIN1JNWn/t6T9f532+W8GkNt7vzz9UkfarZVpb68BuWlbnqW1nU7L2DMx7eCoBlpqGOppCQB83zIgIg+wWQAAAABJRU5ErkJggg==',
             'options': [
                 {
                     'name': 'enabled',
@@ -222,6 +227,14 @@ config = [{
                     'default': 1,
                     'description': 'Favors Golden Popcorn-releases over all other releases.'
                 },
+                {
+                    'name': 'prefer_freeleech',
+                    'advanced': True,
+                    'type': 'bool',
+                    'label': 'Prefer Freeleech',
+                    'default': 1,
+                    'description': 'Favors torrents marked as freeleech over all other releases.'
+                },
                 {
                     'name': 'prefer_scene',
                     'advanced': True,

couchpotato/core/media/_base/providers/torrent/publichd.py

@@ -1,136 +0,0 @@
-from urlparse import parse_qs
-import re
-import traceback
-
-from bs4 import BeautifulSoup
-from couchpotato.core.helpers.encoding import tryUrlencode, toUnicode
-from couchpotato.core.helpers.variable import tryInt
-from couchpotato.core.logger import CPLog
-from couchpotato.core.media._base.providers.torrent.base import TorrentMagnetProvider
-import six
-
-
-log = CPLog(__name__)
-
-
-class Base(TorrentMagnetProvider):
-
-    urls = {
-        'test': 'https://publichd.se',
-        'detail': 'https://publichd.se/index.php?page=torrent-details&id=%s',
-        'search': 'https://publichd.se/index.php',
-    }
-    http_time_between_calls = 0
-
-    def search(self, movie, quality):
-
-        if not quality.get('hd', False):
-            return []
-
-        return super(Base, self).search(movie, quality)
-
-    def _search(self, media, quality, results):
-
-        query = self.buildUrl(media)
-
-        params = tryUrlencode({
-            'page': 'torrents',
-            'search': query,
-            'active': 1,
-        })
-
-        data = self.getHTMLData('%s?%s' % (self.urls['search'], params))
-
-        if data:
-
-            try:
-                soup = BeautifulSoup(data)
-
-                results_table = soup.find('table', attrs = {'id': 'bgtorrlist2'})
-                entries = results_table.find_all('tr')
-
-                for result in entries[2:len(entries) - 1]:
-                    info_url = result.find(href = re.compile('torrent-details'))
-                    download = result.find(href = re.compile('magnet:'))
-
-                    if info_url and download:
-
-                        url = parse_qs(info_url['href'])
-
-                        results.append({
-                            'id': url['id'][0],
-                            'name': six.text_type(info_url.string),
-                            'url': download['href'],
-                            'detail_url': self.urls['detail'] % url['id'][0],
-                            'size': self.parseSize(result.find_all('td')[7].string),
-                            'seeders': tryInt(result.find_all('td')[4].string),
-                            'leechers': tryInt(result.find_all('td')[5].string),
-                            'get_more_info': self.getMoreInfo
-                        })
-
-            except:
-                log.error('Failed getting results from %s: %s', (self.getName(), traceback.format_exc()))
-
-    def getMoreInfo(self, item):
-
-        cache_key = 'publichd.%s' % item['id']
-        description = self.getCache(cache_key)
-
-        if not description:
-
-            try:
-                full_description = self.urlopen(item['detail_url'])
-                html = BeautifulSoup(full_description)
-                nfo_pre = html.find('div', attrs = {'id': 'torrmain'})
-                description = toUnicode(nfo_pre.text) if nfo_pre else ''
-            except:
-                log.error('Failed getting more info for %s', item['name'])
-                description = ''
-
-            self.setCache(cache_key, description, timeout = 25920000)
-
-        item['description'] = description
-        return item
-
-
-config = [{
-    'name': 'publichd',
-    'groups': [
-        {
-            'tab': 'searcher',
-            'list': 'torrent_providers',
-            'name': 'PublicHD',
-            'description': 'Public Torrent site with only HD content. See <a href="https://publichd.se/">PublicHD</a>',
-            'wizard': True,
-            'options': [
-                {
-                    'name': 'enabled',
-                    'type': 'enabler',
-                    'default': True,
-                },
-                {
-                    'name': 'seed_ratio',
-                    'label': 'Seed ratio',
-                    'type': 'float',
-                    'default': 1,
-                    'description': 'Will not be (re)moved until this seed ratio is met.',
-                },
-                {
-                    'name': 'seed_time',
-                    'label': 'Seed time',
-                    'type': 'int',
-                    'default': 40,
-                    'description': 'Will not be (re)moved until this seed time (in hours) is met.',
-                },
-                {
-                    'name': 'extra_score',
-                    'advanced': True,
-                    'label': 'Extra Score',
-                    'type': 'int',
-                    'default': 0,
-                    'description': 'Starting score for each release found via this provider.',
-                }
-            ],
-        },
-    ],
-}]

couchpotato/core/media/_base/providers/torrent/sceneaccess.py

@@ -24,9 +24,9 @@ class Base(TorrentProvider):
 
     http_time_between_calls = 1  # Seconds
 
-    def _search(self, media, quality, results):
+    def _searchOnTitle(self, title, media, quality, results):
 
-        url = self.buildUrl(media, quality)
+        url = self.buildUrl(title, media, quality)
         data = self.getHTMLData(url)
 
         if data:
@@ -89,8 +89,9 @@ config = [{
             'tab': 'searcher',
             'list': 'torrent_providers',
             'name': 'SceneAccess',
-            'description': 'See <a href="https://sceneaccess.eu/">SceneAccess</a>',
+            'description': '<a href="https://sceneaccess.eu/">SceneAccess</a>',
             'wizard': True,
+            'icon': 'iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAIAAACQkWg2AAAABnRSTlMAAAAAAABupgeRAAACT0lEQVR4AYVQS0sbURidO3OTmajJ5FElTTOkPmZ01GhHrIq0aoWAj1Vc+A/cuRMXbl24V9SlCGqrLhVFCrooEhCp2BAx0mobTY2kaR7qmOm87EXL1EWxh29xL+c7nPMdgGHYO5bF/gdbefnr6WlbWRnxluMwAB4Z0uEgXa7nwaDL7+/RNPzxbYvb/XJ0FBYVfd/ayh0fQ4qCGEHcm0KLRZUk7Pb2YRJPRwcsKMidnKD3t9VVT3s7BDh+z5FOZ3Vfn3h+Hltfx00mRRSRWFcUmmVNhYVqPn8dj3va2oh+txvcQRVF9ebm1fi4k+dRFbosY5rm4Hk7xxULQnJnx93S4g0EIEEQRoDLo6PrWEw8Pc0eHLwYGopMTDirqlJ7eyhYYGHhfgfHCcKYksZGVB/NcXI2mw6HhZERqrjYTNPHi4tFPh8aJIYIhgPlcCRDoZLW1s75+Z/7+59nZ/OJhLWigqAoKZX6Mjf3dXkZ3pydGYLc4aEoCCkInzQ1fRobS2xuvllaonkedfArnY5OTdGVldBkOADgqq2Nr6z8CIWaJietDHOhKB+HhwFKC6Gnq4ukKJvP9zcSbjYDXbeVlkKzuZBhnnV3e3t6UOmaJO0ODibW1hB1GYkg8R/gup7Z3TVZLJ5AILW9LcZiVpYtYBhw16O3t7cauckyeF9Tgz0ATpL2+nopmWycmbnY2LiKRjFk6/d7+/vRJfl4HGzV1T0UIM43MGBvaIBWK/YvwM5w+IMgGH8tkyEgvIpE7M3Nt6qqZrNyOq1kMmouh455Ggz+BhKY4GEc2CfwAAAAAElFTkSuQmCC',
             'options': [
                 {
                     'name': 'enabled',

couchpotato/core/media/_base/providers/torrent/thepiratebay.py

@@ -24,15 +24,18 @@ class Base(TorrentMagnetProvider):
     http_time_between_calls = 0
 
     proxy_list = [
-        'https://tpb.ipredator.se',
+        'https://dieroschtibay.org',
+        'https://thebay.al',
         'https://thepiratebay.se',
-        'http://pirateproxy.ca',
-        'http://tpb.al',
-        'http://www.tpb.gr',
-        'http://bayproxy.me',
-        'http://proxybay.eu',
-        'http://www.getpirate.com',
-        'http://piratebay.io',
+        'http://thepiratebay.se.net',
+        'http://thebootlegbay.com',
+        'http://tpb.ninja.so',
+        'http://proxybay.fr',
+        'http://pirateproxy.in',
+        'http://piratebay.skey.sk',
+        'http://pirateproxy.be',
+        'http://bayproxy.li',
+        'http://proxybay.pw',
     ]
 
     def _search(self, media, quality, results):
@@ -65,7 +68,7 @@ class Base(TorrentMagnetProvider):
                         pass
 
                     entries = results_table.find_all('tr')
-                    for result in entries[2:]:
+                    for result in entries[1:]:
                         link = result.find(href = re.compile('torrent\/\d+\/'))
                         download = result.find(href = re.compile('magnet:'))
 
@@ -109,7 +112,11 @@ class Base(TorrentMagnetProvider):
         full_description = self.getCache('tpb.%s' % item['id'], item['detail_url'], cache_timeout = 25920000)
         html = BeautifulSoup(full_description)
         nfo_pre = html.find('div', attrs = {'class': 'nfo'})
-        description = toUnicode(nfo_pre.text) if nfo_pre else ''
+        description = ''
+        try:
+            description = toUnicode(nfo_pre.text)
+        except:
+            pass
 
         item['description'] = description
         return item
@@ -122,8 +129,9 @@ config = [{
             'tab': 'searcher',
             'list': 'torrent_providers',
             'name': 'ThePirateBay',
-            'description': 'The world\'s largest bittorrent tracker. See <a href="http://fucktimkuik.org/">ThePirateBay</a>',
+            'description': 'The world\'s largest bittorrent tracker. <a href="http://fucktimkuik.org/">ThePirateBay</a>',
             'wizard': True,
+            'icon': 'iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAAAAAA6mKC9AAAA3UlEQVQY02P4DwT/YADIZvj//7qnozMYODmtAAusZoCDELDAegYGViZhAWZmRoYoqIDupfhNN1M3dTBEggXWMZg9jZRXV77YxhAOFpjDwMAPMoCXmcHsF1SAQZ6bQY2VgUEbKHClcAYzg3mINEO8jSCD478/DPsZmvqWblu1bOmStes3Pp0ezVDF4Gif0Hfx9///74/ObRZ2YNiZ47C8XIRBxFJR0jbSSUud4f9zAQWn8NTuziAt2zy5xIMM/z8LFX0E+fD/x0MRDCeA1v7Z++Y/FDzyvAtyBxIA+h8A8ZKLeT+lJroAAAAASUVORK5CYII=',
             'options': [
                 {
                     'name': 'enabled',

couchpotato/core/media/_base/providers/torrent/torrentbytes.py

@@ -90,8 +90,9 @@ config = [{
             'tab': 'searcher',
             'list': 'torrent_providers',
             'name': 'TorrentBytes',
-            'description': 'See <a href="http://torrentbytes.net">TorrentBytes</a>',
+            'description': '<a href="http://torrentbytes.net">TorrentBytes</a>',
             'wizard': True,
+            'icon': 'iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAMAAAAoLQ9TAAAAeFBMVEUAAAAAAEQAA1QAEmEAKnQALHYAMoEAOokAQpIASYsASZgAS5UATZwATosATpgAVJ0AWZwAYZ4AZKAAaZ8Ab7IAcbMAfccAgcQAgcsAhM4AiscAjMkAmt0AoOIApecAp/EAqvQAs+kAt+wA3P8A4f8A//8VAAAfDbiaAl08AAAAjUlEQVQYGQXBO04DQRAFwHqz7Z8sECIl5f73ISRD5GBs7UxTlWfg9vYXnvJRQJqOL88D6BAwJtMMumHUVCl60aa6H93IrIv0b+157f1lpk+fm87lMWrZH0vncKbXdRUQrRmrh9C6Iwkq6rg4PXZcyXmbizzeV/g+rDra0rGve8jPKLSOJNi2AQAwAGjwD7ApPkEHdtPQAAAAAElFTkSuQmCC',
             'options': [
                 {
                     'name': 'enabled',

couchpotato/core/media/_base/providers/torrent/torrentday.py

@@ -18,9 +18,9 @@ class Base(TorrentProvider):
 
     http_time_between_calls = 1  # Seconds
 
-    def _search(self, media, quality, results):
+    def _searchOnTitle(self, title, media, quality, results):
 
-        query = self.buildUrl(media)
+        query = '"%s" %s' % (title, media['info']['year'])
 
         data = {
             '/browse.php?': None,
@@ -68,8 +68,9 @@ config = [{
             'tab': 'searcher',
             'list': 'torrent_providers',
             'name': 'TorrentDay',
-            'description': 'See <a href="http://www.td.af/">TorrentDay</a>',
+            'description': '<a href="http://www.td.af/">TorrentDay</a>',
             'wizard': True,
+            'icon': 'iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAC5ElEQVQ4y12TXUgUURTH//fO7Di7foeQJH6gEEEIZZllVohfSG/6UA+RSFAQQj74VA8+Bj30lmAlRVSEvZRfhNhaka5ZUG1paKaW39tq5O6Ou+PM3M4o6m6X+XPPzD3zm/+dcy574r515WfIW8CZBM4YAA5Gc/aQC3yd7oXYEONcsISE5dTDh91HS0t7FEWhBUAeN9ynV/d9qJAgE4AECURAcVsGlCCnly26LMA0IQwTa52dje3d3e3hcPi8qqrrMjcVYI3EHCQZlkFOHBwR2QHh2ASAAIJxWGAQEDxjePhs3527XjJwnb37OHBq0T+Tyyjh+9KnEzNJ7nouc1Q/3A3HGsOvnJy+PSUlj81w2Lny9WuJ6+3AmTjD4HOcrdR2dWXLRQePvyaSLfQOPMPC8mC9iHCsOxSyzJCelzdSXlNzD5ujpb25Wbfc/XXJemTXF4+nnCNq+AMLe50uFfEJTiw4GXSFtiHL0SnIq66+p0kSArqO+eH3RdsAv9+f5vW7L7GICq6rmM8XBCAXlBw90rOyxibn5yzfkg/L09M52/jxqdESaIrBXHYZZbB1GX8cEpySxKIB8S5XcOnvqpli1zuwmrTtoLjw5LOK/eeuWsE4JH5IRPaPZKiKigmPp+5pa+u1aEjIMhEgrRkmi9mgxGUhM7LNJSzOzsE3+cOeExovXOjdytE0LV4zqNZUtV0uZzAGoGkhDH/2YHZiErmv4uyWQnZZWc+hoqL3WzlTExN5hhA8IEwkZWZOxwB++30YG/9GkYCPvqAaHAW5uWPROW86OmqCprUR7z1yZDAGQNuCvkoB/baIKUBWMTYymv+gra3eJNvjXu+B562tFyXqTJ6YuHK8rKwvBmC3vR7cOCPQLWFz8LnfXWUrJo9U19BwMyUlJRjTSMJ2ENxUiGxq9KXQfwqYlnWstvbR5aamG9g0uzM8Q4OFt++3NNixQ2NgYmeN03FOTUv7XVpV9aKisvLl1vN/WVhNc/Fi1NEAAAAASUVORK5CYII=',
             'options': [
                 {
                     'name': 'enabled',

couchpotato/core/media/_base/providers/torrent/torrentleech.py

@@ -13,20 +13,20 @@ log = CPLog(__name__)
 class Base(TorrentProvider):
 
     urls = {
-        'test': 'http://www.torrentleech.org/',
-        'login': 'http://www.torrentleech.org/user/account/login/',
-        'login_check': 'http://torrentleech.org/user/messages',
-        'detail': 'http://www.torrentleech.org/torrent/%s',
-        'search': 'http://www.torrentleech.org/torrents/browse/index/query/%s/categories/%d',
-        'download': 'http://www.torrentleech.org%s',
+        'test': 'https://www.torrentleech.org/',
+        'login': 'https://www.torrentleech.org/user/account/login/',
+        'login_check': 'https://torrentleech.org/user/messages',
+        'detail': 'https://www.torrentleech.org/torrent/%s',
+        'search': 'https://www.torrentleech.org/torrents/browse/index/query/%s/categories/%d',
+        'download': 'https://www.torrentleech.org%s',
     }
 
     http_time_between_calls = 1  # Seconds
     cat_backup_id = None
 
-    def _search(self, media, quality, results):
+    def _searchOnTitle(self, title, media, quality, results):
 
-        url = self.urls['search'] % self.buildUrl(media, quality)
+        url = self.urls['search'] % self.buildUrl(title, media, quality)
 
         data = self.getHTMLData(url)
 
@@ -80,8 +80,9 @@ config = [{
             'tab': 'searcher',
             'list': 'torrent_providers',
             'name': 'TorrentLeech',
-            'description': 'See <a href="http://torrentleech.org">TorrentLeech</a>',
+            'description': '<a href="http://torrentleech.org">TorrentLeech</a>',
             'wizard': True,
+            'icon': 'iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAIAAACQkWg2AAACHUlEQVR4AZVSO48SYRSdGTCBEMKzILLAWiybkKAGMZRUUJEoDZX7B9zsbuQPYEEjNLTQkYgJDwsoSaxspEBsCITXjjNAIKi8AkzceXgmbHQ1NJ5iMufmO9/9zrmXlCSJ+B8o75J8Pp/NZj0eTzweBy0Wi4PBYD6f12o1r9ebTCZx+22HcrnMsuxms7m6urTZ7LPZDMVYLBZ8ZV3yo8aq9Pq0wzCMTqe77dDv9y8uLyAWBH6xWOyL0K/56fcb+rrPgPZ6PZfLRe1fsl6vCUmGKIqoqNXqdDr9Dbjps9znUV0uTqdTjuPkDoVCIfcuJ4gizjMMm8u9vW+1nr04czqdK56c37CbKY9j2+1WEARZ0Gq1RFHAz2q1qlQqXxoN69HRcDjUarW8ZD6QUigUOnY8uKYH8N1sNkul9yiGw+F6vS4Rxn8EsodEIqHRaOSnq9T7ajQazWQycEIR1AEBYDabSZJyHDucJyegwWBQr9ebTCaKvHd4cCQANUU9evwQ1Ofz4YvUKUI43GE8HouSiFiNRhOowWBIpVLyHITJkuW3PwgAEf3pgIwxF5r+OplMEsk3CPT5szCMnY7EwUdhwUh/CXiej0Qi3idPz89fdrpdbsfBzH7S3Q9K5pP4c0sAKpVKoVAQGO1ut+t0OoFAQHkH2Da/3/+but3uarWK0ZMQoNdyucRutdttmqZxMTzY7XaYxsrgtUjEZrNhkSwWyy/0NCatZumrNQAAAABJRU5ErkJggg==',
             'options': [
                 {
                     'name': 'enabled',

couchpotato/core/media/_base/providers/torrent/torrentpotato.py

@@ -1,4 +1,4 @@
-from urlparse import urlparse
+from six.moves import urllib
 import re
 import traceback
 
@@ -45,7 +45,7 @@ class Base(TorrentProvider):
                         results.append({
                             'id': torrent.get('torrent_id'),
                             'protocol': 'torrent' if re.match('^(http|https|ftp)://.*$', torrent.get('download_url')) else 'torrent_magnet',
-                            'provider_extra': urlparse(host['host']).hostname or host['host'],
+                            'provider_extra': urllib.urlparse(host['host']).hostname or host['host'],
                             'name': toUnicode(torrent.get('release_name')),
                             'url': torrent.get('download_url'),
                             'detail_url': torrent.get('details_url'),
@@ -134,6 +134,7 @@ config = [{
             'order': 10,
             'description': 'CouchPotato torrent provider. Checkout <a href="https://github.com/RuudBurger/CouchPotatoServer/wiki/CouchPotato-Torrent-Provider">the wiki page about this provider</a> for more info.',
             'wizard': True,
+            'icon': 'iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAIAAACQkWg2AAAABnRSTlMAAAAAAABupgeRAAABSElEQVR4AZ2Nz0oCURTGv8t1YMpqUxt9ARFxoQ/gQtppgvUKcu/sxB5iBJkogspaBC6iVUplEC6kv+oiiKDNhAtt16roP0HQgdsMLgaxfvy4nHP4Pi48qE2g4v91JOqT1CH/UnA7w7icUlLawyEdj+ZI/7h6YluWbRiddHonHh9M70aj7VTKzuXuikUMci/EO/ACnAI15599oAk8AR/AgxBQNCzreD7bmpl+FOIVuAHqQDUcJo+AK+CZFKLt95/MpSmMt0TiW9POxse6UvYZ6zB2wFgjFiNpOGesR0rZ0PVPXf8KhUCl22CwClz4eN8weoZBb9c0bdPsOWvHx/cYu9Y0CoNoZTJrwAbn5DrnZc6XOV+igVbnsgo0IxEomlJuA1vUIYGyq3PZBChwmExCUSmVZgMBDIUCK4UCFIv5vHIhm/XUDeAf/ADbcpd5+aXSWQAAAABJRU5ErkJggg==',
             'options': [
                 {
                     'name': 'enabled',

couchpotato/core/media/_base/providers/torrent/torrentshack.py

@@ -13,12 +13,12 @@ log = CPLog(__name__)
 class Base(TorrentProvider):
 
     urls = {
-        'test': 'https://torrentshack.net/',
-        'login': 'https://torrentshack.net/login.php',
-        'login_check': 'https://torrentshack.net/inbox.php',
-        'detail': 'https://torrentshack.net/torrent/%s',
-        'search': 'https://torrentshack.net/torrents.php?action=advanced&searchstr=%s&scene=%s&filter_cat[%d]=1',
-        'download': 'https://torrentshack.net/%s',
+        'test': 'http://torrentshack.eu/',
+        'login': 'http://torrentshack.eu/login.php',
+        'login_check': 'http://torrentshack.eu/inbox.php',
+        'detail': 'http://torrentshack.eu/torrent/%s',
+        'search': 'http://torrentshack.eu/torrents.php?action=advanced&searchstr=%s&scene=%s&filter_cat[%d]=1',
+        'download': 'http://torrentshack.eu/%s',
     }
 
     http_time_between_calls = 1  # Seconds
@@ -42,15 +42,16 @@ class Base(TorrentProvider):
 
                     link = result.find('span', attrs = {'class': 'torrent_name_link'}).parent
                     url = result.find('td', attrs = {'class': 'torrent_td'}).find('a')
+                    tds = result.find_all('td')
 
                     results.append({
                         'id': link['href'].replace('torrents.php?torrentid=', ''),
                         'name': six.text_type(link.span.string).translate({ord(six.u('\xad')): None}),
                         'url': self.urls['download'] % url['href'],
                         'detail_url': self.urls['download'] % link['href'],
-                        'size': self.parseSize(result.find_all('td')[4].string),
-                        'seeders': tryInt(result.find_all('td')[6].string),
-                        'leechers': tryInt(result.find_all('td')[7].string),
+                        'size': self.parseSize(result.find_all('td')[5].string),
+                        'seeders': tryInt(tds[len(tds)-2].string),
+                        'leechers': tryInt(tds[len(tds)-1].string),
                     })
 
             except:
@@ -80,7 +81,9 @@ config = [{
             'tab': 'searcher',
             'list': 'torrent_providers',
             'name': 'TorrentShack',
-            'description': 'See <a href="https://www.torrentshack.net/">TorrentShack</a>',
+            'description': '<a href="http://torrentshack.eu/">TorrentShack</a>',
+            'wizard': True,
+            'icon': 'iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAIAAACQkWg2AAABmElEQVQoFQXBzY2cVRiE0afqvd84CQiAnxWWtyxsS6ThINBYg2Dc7mZBMEjE4mzs6e9WcY5+ePNuVFJJodQAoLo+SaWCy9rcV8cmjah3CI6iYu7oRU30kE5xxELRfamklY3k1NL19sSm7vPzP/ZdNZzKVDaY2sPZJBh9fv5ITrmG2+Vp4e1sPchVqTCQZJnVXi+/L4uuAJGly1+Pw8CprLbi8Om7tbT19/XRqJUk11JP9uHj9ulxhXbvJbI9qJvr5YkGXFG2IBT8tXczt+sfzDZCp3765f3t9tHEHGEDACma77+8o4oATKk+/PfW9YmHruRFjWoVSFsVsGu1YSKq6Oc37+n98unPZSRlY7vsKDqN+92X3yR9+PdXee3iJNKMStqdcZqoTJbUSi5JOkpfRlhSI0mSpEmCFKoU7FqSNOLAk54uGwCStMUCgLrVic62g7oDoFmmdI+P3S0pDe1xvDqb6XrZqbtzShWNoh9fv/XQHaDdM9OqrZi2M7M3UrB2vlkPS1IbdEBk7UiSoD6VlZ6aKWer4aH4f/AvKoHUTjuyAAAAAElFTkSuQmCC',
             'options': [
                 {
                     'name': 'enabled',

couchpotato/core/media/_base/providers/torrent/torrentz.py

@@ -80,11 +80,12 @@ config = [{
             'name': 'Torrentz',
             'description': 'Torrentz is a free, fast and powerful meta-search engine. <a href="https://torrentz.eu/">Torrentz</a>',
             'wizard': True,
+            'icon': 'iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAAQklEQVQ4y2NgAALjtJn/ycEMlGiGG0IVAxiwAKzOxaKGARcgxgC8YNSAwWoAzuRMjgsIugqfAUR5CZcBRIcHsWEAADSA96Ig020yAAAAAElFTkSuQmCC',
             'options': [
                 {
                     'name': 'enabled',
                     'type': 'enabler',
-                    'default': False
+                    'default': True
                 },
                 {
                     'name': 'verified_only',

couchpotato/core/media/_base/providers/torrent/yify.py

@@ -2,13 +2,13 @@ import traceback
 
 from couchpotato.core.helpers.variable import tryInt, getIdentifier
 from couchpotato.core.logger import CPLog
-from couchpotato.core.media._base.providers.torrent.base import TorrentProvider
+from couchpotato.core.media._base.providers.torrent.base import TorrentMagnetProvider
 
 
 log = CPLog(__name__)
 
 
-class Base(TorrentProvider):
+class Base(TorrentMagnetProvider):
 
     urls = {
         'test': '%s/api',
@@ -35,7 +35,11 @@ class Base(TorrentProvider):
 
     def _search(self, movie, quality, results):
 
-        search_url = self.urls['search'] % (self.getDomain(), getIdentifier(movie), quality['identifier'])
+        domain = self.getDomain()
+        if not domain:
+            return
+
+        search_url = self.urls['search'] % (domain, getIdentifier(movie), quality['identifier'])
 
         data = self.getJsonData(search_url)
 
@@ -43,21 +47,19 @@ class Base(TorrentProvider):
             try:
                 for result in data.get('MovieList'):
 
-                    try:
-                        title = result['TorrentUrl'].split('/')[-1][:-8].replace('_', '.').strip('._')
-                        title = title.replace('.-.', '-')
-                        title = title.replace('..', '.')
-                    except:
-                        continue
+                    if result['Quality'] and result['Quality'] not in result['MovieTitle']:
+                        title = result['MovieTitle'] + ' BrRip ' + result['Quality']
+                    else:
+                        title = result['MovieTitle'] + ' BrRip'
 
                     results.append({
                         'id': result['MovieID'],
                         'name': title,
                         'url': result['TorrentMagnetUrl'],
-                        'detail_url': self.urls['detail'] % (self.getDomain(), result['MovieID']),
+                        'detail_url': self.urls['detail'] % (domain, result['MovieID']),
                         'size': self.parseSize(result['Size']),
                         'seeders': tryInt(result['TorrentSeeds']),
-                        'leechers': tryInt(result['TorrentPeers'])
+                        'leechers': tryInt(result['TorrentPeers']),
                     })
 
             except:
@@ -77,6 +79,7 @@ config = [{
             'name': 'Yify',
             'description': 'Free provider, less accurate. Small HD movies, encoded by <a href="https://yify-torrents.com/">Yify</a>.',
             'wizard': False,
+            'icon': 'iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAIAAACQkWg2AAACL0lEQVR4AS1SPW/UQBAd23fxne/Ld2dvzvHuzPocEBAKokCBqGiQ6IgACYmvUKRBFEQgKKGg4BAlUoggggYUEQpSHOI7CIEoQs/fYcbLaU/efTvvvZlnA1qydoxU5kcxX0CkgmQZtPy0hCUjvK+WgEByOZ5dns1O5bzna8fRVkgsxH8B0YouIvBhdD5T11NiVOoKrsttyUcpRW0InUrFnwe9HzuP2uaQZYhF2LQ76TTXw2RVMTK8mYYbjfh+zNquMVCrqn93aArLSixPxnafdGDLaz1tjY5rmNa8z5BczEQOxQfCl1GyoqoWxYRN1bkh7ELw3q/vhP6HIL4TG9KumpjgvwuyM7OsjSj98E/vszMfZ7xvPtMaWxGO5crwIumKCR5HxDtJ0AWKGG204RfUd/3smJYqwem/Q7BTS1ZGfM4LNpVwuKAz6cMeROst0S2EwNE7GjTehO2H3dxqIpdkydat15G3F8SXBi4GlpBNlSz012L/k2+W0CLLk/jbcf13rf41yJeMQ8QWUZiHCfCA9ad+81nEKPtoS9mJOf9v0NmMJHgUT6xayheK9EIK7JJeU/AF4scDF7Y5SPlJrRcxJ+um4ibNEdObxLiIwJim+eT2AL5D9CIcnZ5zvSJi9eIlNHVVtZ831dk5svPgvjPWTq+ktWkd/kD0qtm71x+sDQe3kt6DXnM7Ct+GajmTxKlkAokWljyAKSm5oWa2w+BH4P2UuVub7eTyiGOQYapY/wEztHduSDYz5gAAAABJRU5ErkJggg==',
             'options': [
                 {
                     'name': 'enabled',

couchpotato/core/media/_base/providers/userscript/base.py

@@ -1,4 +1,4 @@
-from urlparse import urlparse
+from six.moves import urllib
 
 from couchpotato.core.event import addEvent, fireEvent
 from couchpotato.core.helpers.encoding import simplifyString
@@ -34,7 +34,7 @@ class UserscriptBase(Plugin):
 
     def belongsTo(self, url):
 
-        host = urlparse(url).hostname
+        host = urllib.urlparse(url).hostname
         host_split = host.split('.')
         if len(host_split) > 2:
             host = host[len(host_split[0]):]

couchpotato/core/media/_base/search/main.py

@@ -1,8 +1,9 @@
 from couchpotato.api import addApiView
 from couchpotato.core.event import fireEvent, addEvent
-from couchpotato.core.helpers.variable import mergeDicts
+from couchpotato.core.helpers.variable import mergeDicts, getImdb
 from couchpotato.core.logger import CPLog
 from couchpotato.core.plugins.base import Plugin
+import six
 
 log = CPLog(__name__)
 
@@ -30,17 +31,26 @@ class Search(Plugin):
     def search(self, q = '', types = None, **kwargs):
 
         # Make sure types is the correct instance
-        if isinstance(types, (str, unicode)):
+        if isinstance(types, six.string_types):
             types = [types]
         elif isinstance(types, (list, tuple, set)):
             types = list(types)
 
+        imdb_identifier = getImdb(q)
+
         if not types:
-            result = fireEvent('info.search', q = q, merge = True)
+            if imdb_identifier:
+                result = fireEvent('movie.info', identifier = imdb_identifier, merge = True)
+                result = {result['type']: [result]}
+            else:
+                result = fireEvent('info.search', q = q, merge = True)
         else:
             result = {}
             for media_type in types:
-                result[media_type] = fireEvent('%s.search' % media_type)
+                if imdb_identifier:
+                    result[media_type] = fireEvent('%s.info' % media_type, identifier = imdb_identifier)
+                else:
+                    result[media_type] = fireEvent('%s.search' % media_type, q = q)
 
         return mergeDicts({
             'success': True,

couchpotato/core/media/_base/search/static/search.css

@@ -7,7 +7,6 @@
 	text-align: right;
 	height: 100%;
 	transition: all .4s cubic-bezier(0.9,0,0.1,1);
-	position: absolute;
 	z-index: 20;
 	border: 0 solid transparent;
 	border-bottom-width: 4px;

couchpotato/core/media/_base/search/static/search.js

@@ -13,6 +13,9 @@ Block.Search = new Class({
 				self.input = new Element('input', {
 					'placeholder': 'Search & add a new media',
 					'events': {
+						'input': self.keyup.bind(self),
+						'paste': self.keyup.bind(self),
+						'change': self.keyup.bind(self),
 						'keyup': self.keyup.bind(self),
 						'focus': function(){
 							if(focus_timer) clearTimeout(focus_timer);

couchpotato/core/media/_base/searcher/__init__.py

@@ -73,4 +73,24 @@ config = [{
             ],
         },
     ],
+}, {
+    'name': 'torrent',
+    'groups': [
+        {
+            'tab': 'searcher',
+            'name': 'searcher',
+            'wizard': True,
+            'options': [
+                {
+                    'name': 'minimum_seeders',
+                    'advanced': True,
+                    'label': 'Minimum seeders',
+                    'description': 'Ignore torrents with seeders below this number',
+                    'default': 1,
+                    'type': 'int',
+                    'unit': 'seeders'
+                },
+            ],
+        },
+    ],
 }]

couchpotato/core/media/_base/searcher/main.py

@@ -87,31 +87,23 @@ class Searcher(SearcherBase):
     def containsOtherQuality(self, nzb, movie_year = None, preferred_quality = None):
         if not preferred_quality: preferred_quality = {}
 
-        name = nzb['name']
-        size = nzb.get('size', 0)
-        nzb_words = re.split('\W+', simplifyString(name))
-
-        qualities = fireEvent('quality.all', single = True)
-
         found = {}
-        for quality in qualities:
-            # Main in words
-            if quality['identifier'] in nzb_words:
-                found[quality['identifier']] = True
-
-            # Alt in words
-            if list(set(nzb_words) & set(quality['alternative'])):
-                found[quality['identifier']] = True
 
         # Try guessing via quality tags
-        guess = fireEvent('quality.guess', [nzb.get('name')], single = True)
+        guess = fireEvent('quality.guess', files = [nzb.get('name')], size = nzb.get('size', None), single = True)
         if guess:
             found[guess['identifier']] = True
 
         # Hack for older movies that don't contain quality tag
+        name = nzb['name']
+        size = nzb.get('size', 0)
+
         year_name = fireEvent('scanner.name_year', name, single = True)
         if len(found) == 0 and movie_year < datetime.datetime.now().year - 3 and not year_name.get('year', None):
-            if size > 3000:  # Assume dvdr
+            if size > 20000:  # Assume bd50
+                log.info('Quality was missing in name, assuming it\'s a BR-Disk based on the size: %s', size)
+                found['bd50'] = True
+            elif size > 3000:  # Assume dvdr
                 log.info('Quality was missing in name, assuming it\'s a DVD-R based on the size: %s', size)
                 found['dvdr'] = True
             else:  # Assume dvdrip
@@ -123,7 +115,10 @@ class Searcher(SearcherBase):
             if found.get(allowed):
                 del found[allowed]
 
-        return not (found.get(preferred_quality['identifier']) and len(found) == 1)
+        if found.get(preferred_quality['identifier']) and len(found) == 1:
+            return False
+
+        return found
 
     def correct3D(self, nzb, preferred_quality = None):
         if not preferred_quality: preferred_quality = {}
@@ -134,7 +129,11 @@ class Searcher(SearcherBase):
         # Try guessing via quality tags
         guess = fireEvent('quality.guess', [nzb.get('name')], single = True)
 
-        return threed == guess.get('is_3d')
+        if guess:
+            return threed == guess.get('is_3d')
+        # If no quality guess, assume not 3d
+        else:
+            return threed == False
 
     def correctYear(self, haystack, year, year_range):
 
@@ -179,6 +178,25 @@ class Searcher(SearcherBase):
 
         return False
 
+    def containsWords(self, rel_name, rel_words, conf, media):
+
+        # Make sure it has required words
+        words = splitString(self.conf('%s_words' % conf, section = 'searcher').lower())
+        try: words = removeDuplicate(words + splitString(media['category'][conf].lower()))
+        except: pass
+
+        req_match = 0
+        for req_set in words:
+            if len(req_set) >= 2 and (req_set[:1] + req_set[-1:]) == '//':
+                if re.search(req_set[1:-1], rel_name):
+                    log.debug('Regex match: %s', req_set[1:-1])
+                    req_match += 1
+            else:
+                req = splitString(req_set, '&')
+                req_match += len(list(set(rel_words) & set(req))) == len(req)
+
+        return words, req_match > 0
+
     def correctWords(self, rel_name, media):
         media_title = fireEvent('searcher.get_search_title', media, single = True)
         media_words = re.split('\W+', simplifyString(media_title))
@@ -186,31 +204,13 @@ class Searcher(SearcherBase):
         rel_name = simplifyString(rel_name)
         rel_words = re.split('\W+', rel_name)
 
-        # Make sure it has required words
-        required_words = splitString(self.conf('required_words', section = 'searcher').lower())
-        try: required_words = removeDuplicate(required_words + splitString(media['category']['required'].lower()))
-        except: pass
-
-        req_match = 0
-        for req_set in required_words:
-            req = splitString(req_set, '&')
-            req_match += len(list(set(rel_words) & set(req))) == len(req)
-
-        if len(required_words) > 0 and req_match == 0:
+        required_words, contains_required = self.containsWords(rel_name, rel_words, 'required', media)
+        if len(required_words) > 0 and not contains_required:
             log.info2('Wrong: Required word missing: %s', rel_name)
             return False
 
-        # Ignore releases
-        ignored_words = splitString(self.conf('ignored_words', section = 'searcher').lower())
-        try: ignored_words = removeDuplicate(ignored_words + splitString(media['category']['ignored'].lower()))
-        except: pass
-
-        ignored_match = 0
-        for ignored_set in ignored_words:
-            ignored = splitString(ignored_set, '&')
-            ignored_match += len(list(set(rel_words) & set(ignored))) == len(ignored)
-
-        if len(ignored_words) > 0 and ignored_match:
+        ignored_words, contains_ignored = self.containsWords(rel_name, rel_words, 'ignored', media)
+        if len(ignored_words) > 0 and contains_ignored:
             log.info2("Wrong: '%s' contains 'ignored words'", rel_name)
             return False
 

couchpotato/core/media/movie/_base/main.py

@@ -1,10 +1,10 @@
-import os
 import traceback
 import time
 
 from couchpotato import get_db
 from couchpotato.api import addApiView
 from couchpotato.core.event import fireEvent, fireEventAsync, addEvent
+from couchpotato.core.helpers.database import RecordNotFound
 from couchpotato.core.helpers.encoding import toUnicode
 from couchpotato.core.helpers.variable import splitString, getTitle, getImdb, getIdentifier
 from couchpotato.core.logger import CPLog
@@ -27,6 +27,10 @@ class MovieBase(MovieTypeBase):
 
         addApiView('movie.add', self.addView, docs = {
             'desc': 'Add new movie to the wanted list',
+            'return': {'type': 'object', 'example': """{
+    'success': True,
+    'movie': object
+}"""},
             'params': {
                 'identifier': {'desc': 'IMDB id of the movie your want to add.'},
                 'profile_id': {'desc': 'ID of quality profile you want the add the movie in. If empty will use the default profile.'},
@@ -45,7 +49,7 @@ class MovieBase(MovieTypeBase):
         })
 
         addEvent('movie.add', self.add)
-        addEvent('movie.update_info', self.updateInfo)
+        addEvent('movie.update', self.update)
         addEvent('movie.update_release_dates', self.updateReleaseDate)
 
     def add(self, params = None, force_readd = True, search_after = True, update_after = True, notify_after = True, status = None):
@@ -90,7 +94,7 @@ class MovieBase(MovieTypeBase):
 
         # Default profile and category
         default_profile = {}
-        if not params.get('profile_id'):
+        if (not params.get('profile_id') and status != 'done') or params.get('ignore_previous', False):
             default_profile = fireEvent('profile.default', single = True)
         cat_id = params.get('category_id')
 
@@ -105,7 +109,7 @@ class MovieBase(MovieTypeBase):
                     'imdb': params.get('identifier')
                 },
                 'status': status if status else 'active',
-                'profile_id': params.get('profile_id', default_profile.get('_id')),
+                'profile_id': params.get('profile_id') or default_profile.get('_id'),
                 'category_id': cat_id if cat_id is not None and len(cat_id) > 0 and cat_id != '-1' else None,
             }
 
@@ -117,8 +121,17 @@ class MovieBase(MovieTypeBase):
             media['info'] = info
 
             new = False
+            previous_profile = None
             try:
                 m = db.get('media', 'imdb-%s' % params.get('identifier'), with_doc = True)['doc']
+
+                try:
+                    db.get('id', m.get('profile_id'))
+                    previous_profile = m.get('profile_id')
+                except RecordNotFound:
+                    pass
+                except:
+                    log.error('Failed getting previous profile: %s', traceback.format_exc())
             except:
                 new = True
                 m = db.insert(media)
@@ -139,16 +152,16 @@ class MovieBase(MovieTypeBase):
 
                 # Clean snatched history
                 for release in fireEvent('release.for_media', m['_id'], single = True):
-                    if release.get('status') in ['downloaded', 'snatched', 'done']:
+                    if release.get('status') in ['downloaded', 'snatched', 'seeding', 'done']:
                         if params.get('ignore_previous', False):
-                            release['status'] = 'ignored'
-                            db.update(release)
+                            fireEvent('release.update_status', release['_id'], status = 'ignored')
                         else:
                             fireEvent('release.delete', release['_id'], single = True)
 
-                m['profile_id'] = params.get('profile_id', default_profile.get('id'))
+                m['profile_id'] = (params.get('profile_id') or default_profile.get('_id')) if not previous_profile else previous_profile
                 m['category_id'] = cat_id if cat_id is not None and len(cat_id) > 0 else (m.get('category_id') or None)
                 m['last_edit'] = int(time.time())
+                m['tags'] = []
 
                 do_search = True
                 db.update(m)
@@ -161,7 +174,7 @@ class MovieBase(MovieTypeBase):
             # Trigger update info
             if added and update_after:
                 # Do full update to get images etc
-                fireEventAsync('movie.update_info', m['_id'], default_title = params.get('title'), on_complete = onComplete)
+                fireEventAsync('movie.update', m['_id'], default_title = params.get('title'), on_complete = onComplete)
 
             # Remove releases
             for rel in fireEvent('release.for_media', m['_id'], single = True):
@@ -169,6 +182,9 @@ class MovieBase(MovieTypeBase):
                     db.delete(rel)
 
             movie_dict = fireEvent('media.get', m['_id'], single = True)
+            if not movie_dict:
+                log.debug('Failed adding media, can\'t find it anymore')
+                return False
 
             if do_search and search_after:
                 onComplete = self.createOnComplete(m['_id'])
@@ -225,7 +241,7 @@ class MovieBase(MovieTypeBase):
 
                     db.update(m)
 
-                    fireEvent('media.restatus', m['_id'])
+                    fireEvent('media.restatus', m['_id'], single = True)
 
                     m = db.get('id', media_id)
 
@@ -245,7 +261,7 @@ class MovieBase(MovieTypeBase):
             'success': False,
         }
 
-    def updateInfo(self, media_id = None, identifier = None, default_title = None, extended = False):
+    def update(self, media_id = None, identifier = None, default_title = None, extended = False):
         """
         Update movie information inside media['doc']['info']
 
@@ -258,6 +274,10 @@ class MovieBase(MovieTypeBase):
         if self.shuttingDown():
             return
 
+        lock_key = 'media.get.%s' % media_id if media_id else identifier
+        self.acquireLock(lock_key)
+
+        media = {}
         try:
             db = get_db()
 
@@ -301,42 +321,16 @@ class MovieBase(MovieTypeBase):
                 media['title'] = def_title
 
             # Files
-            images = info.get('images', [])
-            media['files'] = media.get('files', {})
-            for image_type in ['poster']:
+            image_urls = info.get('images', [])
 
-                # Remove non-existing files
-                file_type = 'image_%s' % image_type
-                existing_files = list(set(media['files'].get(file_type, [])))
-                for ef in media['files'].get(file_type, []):
-                    if not os.path.isfile(ef):
-                        existing_files.remove(ef)
-
-                # Replace new files list
-                media['files'][file_type] = existing_files
-                if len(existing_files) == 0:
-                    del media['files'][file_type]
-
-                # Loop over type
-                for image in images.get(image_type, []):
-                    if not isinstance(image, (str, unicode)):
-                        continue
-
-                    if file_type not in media['files'] or len(media['files'].get(file_type, [])) == 0:
-                        file_path = fireEvent('file.download', url = image, single = True)
-                        if file_path:
-                            media['files'][file_type] = [file_path]
-                            break
-                    else:
-                        break
+            self.getPoster(media, image_urls)
 
             db.update(media)
-
-            return media
         except:
             log.error('Failed update media: %s', traceback.format_exc())
 
-        return {}
+        self.releaseLock(lock_key)
+        return media
 
     def updateReleaseDate(self, media_id):
         """
@@ -352,7 +346,7 @@ class MovieBase(MovieTypeBase):
             media = db.get('id', media_id)
 
             if not media.get('info'):
-                media = self.updateInfo(media_id)
+                media = self.update(media_id)
                 dates = media.get('info', {}).get('release_date')
             else:
                 dates = media.get('info').get('release_date')

couchpotato/core/media/movie/_base/static/list.js

@@ -373,6 +373,7 @@ var MovieList = new Class({
 					(e).preventDefault();
 					this.set('text', 'Deleting..');
 					Api.request('media.delete', {
+						'method': 'post',
 						'data': {
 							'id': ids.join(','),
 							'delete_from': self.options.identifier
@@ -413,6 +414,7 @@ var MovieList = new Class({
 		var ids = self.getSelectedMovies();
 
 		Api.request('movie.edit', {
+			'method': 'post',
 			'data': {
 				'id': ids.join(','),
 				'profile_id': self.mass_edit_quality.get('value')
@@ -426,6 +428,7 @@ var MovieList = new Class({
 		var ids = self.getSelectedMovies();
 
 		Api.request('media.refresh', {
+			'method': 'post',
 			'data': {
 				'id': ids.join(',')
 			}

couchpotato/core/media/movie/_base/static/manage.js

@@ -2,6 +2,7 @@ Page.Manage = new Class({
 
 	Extends: PageBase,
 
+	order: 20,
 	name: 'manage',
 	title: 'Do stuff to your existing movies!',
 

couchpotato/core/media/movie/_base/static/movie.actions.js

@@ -78,7 +78,7 @@ MA.IMDB = new Class({
 	create: function(){
 		var self = this;
 
-		self.id = self.movie.get('imdb') || self.movie.get('identifier');
+		self.id = self.movie.getIdentifier ? self.movie.getIdentifier() : self.get('imdb');
 
 		self.el = new Element('a.imdb', {
 			'title': 'Go to the IMDB page of ' + self.getTitle(),
@@ -115,8 +115,15 @@ MA.Release = new Class({
 
 			self.releases = null;
 			if(self.options_container){
-				self.options_container.destroy();
-				self.options_container = null;
+				// Releases are currently displayed
+				if(self.options_container.isDisplayed()){
+					self.options_container.destroy();
+					self.createReleases();
+				}
+				else {
+					self.options_container.destroy();
+					self.options_container = null;
+				}
 			}
 		});
 
@@ -131,10 +138,10 @@ MA.Release = new Class({
 
 	},
 
-	createReleases: function(){
+	createReleases: function(refresh){
 		var self = this;
 
-		if(!self.options_container){
+		if(!self.options_container || refresh){
 			self.options_container = new Element('div.options').grab(
 				self.release_container = new Element('div.releases.table')
 			);
@@ -252,7 +259,7 @@ MA.Release = new Class({
 
 				self.trynext_container.adopt(
 					new Element('span.or', {
-						'text': 'This movie is snatched, if anything went wrong, download'
+						'text': 'If anything went wrong, download'
 					}),
 					lr ? new Element('a.button.orange', {
 						'text': 'the same release again',
@@ -302,7 +309,7 @@ MA.Release = new Class({
 			self.movie.data.releases.each(function(release){
 				if(has_available && has_snatched) return;
 
-				if(['snatched', 'downloaded', 'seeding'].contains(release.status))
+				if(['snatched', 'downloaded', 'seeding', 'done'].contains(release.status))
 					has_snatched = true;
 
 				if(['available'].contains(release.status))
@@ -684,7 +691,7 @@ MA.Readd = new Class({
 		var movie_done = self.movie.data.status == 'done';
 		if(self.movie.data.releases && !movie_done)
 			var snatched = self.movie.data.releases.filter(function(release){
-				return release.status && (release.status == 'snatched' || release.status == 'downloaded' || release.status == 'done');
+				return release.status && (release.status == 'snatched' || release.status == 'seeding' || release.status == 'downloaded' || release.status == 'done');
 			}).length;
 
 		if(movie_done || snatched && snatched > 0)
@@ -703,7 +710,7 @@ MA.Readd = new Class({
 
 		Api.request('movie.add', {
 			'data': {
-				'identifier': self.movie.get('identifier'),
+				'identifier': self.movie.getIdentifier(),
 				'ignore_previous': 1
 			}
 		});

couchpotato/core/media/movie/_base/static/movie.css

@@ -123,6 +123,7 @@
 		.movies.thumbs_list .movie {
 			width: 16.66667%;
 			height: auto;
+			min-height: 200px;
 			display: inline-block;
 			margin: 0;
 			padding: 0;
@@ -133,6 +134,7 @@
 			@media all and (max-width: 800px) {
 				.movies.thumbs_list .movie {
 					width: 25%;
+					min-height: 100px;
 				}
 			}
 
@@ -355,18 +357,40 @@
 				top: 30px;
 				clear: both;
 				bottom: 30px;
-				overflow: hidden;
 				position: absolute;
 			}
-				.movies .data:hover .description {
-					overflow: auto;
-				}
 				.movies.list_list .movie:not(.details_view) .info .description,
 				.movies.mass_edit_list .info .description,
 				.movies.thumbs_list .info .description {
 					display: none;
 				}
 
+		.movies .data .eta {
+			display: none;
+		}
+
+		.movies.details_list .data .eta {
+			position: absolute;
+			bottom: 0;
+			right: 0;
+			display: block;
+			min-height: 20px;
+			text-align: right;
+			font-style: italic;
+			opacity: .8;
+			font-size: 11px;
+		}
+
+			.movies.details_list .movie:hover .data .eta {
+				display: none;
+			}
+
+		.movies.thumbs_list .data .eta {
+			display: block;
+			position: absolute;
+			bottom: 40px;
+		}
+
 		.movies .data .quality {
 			position: absolute;
 			bottom: 2px;

couchpotato/core/media/movie/_base/static/movie.js

@@ -54,13 +54,21 @@ var Movie = new Class({
 		// Reload when releases have updated
 		self.global_events['release.update_status'] = function(notification){
 			var data = notification.data;
-			if(data && self.data._id == data.movie_id){
+			if(data && self.data._id == data.media_id){
 
 				if(!self.data.releases)
 					self.data.releases = [];
 
-				self.data.releases.push({'quality': data.quality, 'status': data.status});
-				self.updateReleases();
+				var updated = false;
+				self.data.releases.each(function(release){
+					if(release._id == data._id){
+						release['status'] = data.status;
+						updated = true;
+					}
+				});
+
+				if(updated)
+					self.updateReleases();
 			}
 		};
 
@@ -136,6 +144,21 @@ var Movie = new Class({
 
 		self.el.addClass('status_'+self.get('status'));
 
+		var eta = null,
+			eta_date = null,
+			now = Math.round(+new Date()/1000);
+
+		if(self.data.info.release_date)
+			[self.data.info.release_date.dvd, self.data.info.release_date.theater].each(function(timestamp){
+				if (timestamp > 0 && (eta == null || Math.abs(timestamp - now) < Math.abs(eta - now)))
+					eta = timestamp;
+			});
+
+		if(eta){
+			eta_date = new Date(eta * 1000);
+			eta_date = eta_date.toLocaleString('en-us', { month: "long" }) + ' ' + eta_date.getFullYear();
+		}
+
 		self.el.adopt(
 			self.select_checkbox = new Element('input[type=checkbox].inlay', {
 				'events': {
@@ -144,7 +167,7 @@ var Movie = new Class({
 					}
 				}
 			}),
-			self.thumbnail = (self.data.files && self.data.files.image_poster) ? new Element('img', {
+			self.thumbnail = (self.data.files && self.data.files.image_poster && self.data.files.image_poster.length > 0) ? new Element('img', {
 				'class': 'type_image poster',
 				'src': Api.createUrl('file.cache') + self.data.files.image_poster[0].split(Api.getOption('path_sep')).pop()
 			}): null,
@@ -158,9 +181,13 @@ var Movie = new Class({
 							'text': self.data.info.year || 'n/a'
 						})
 					),
-					self.description = new Element('div.description', {
+					self.description = new Element('div.description.tiny_scroll', {
 						'text': self.data.info.plot
 					}),
+					self.eta = eta_date && (now+8035200 > eta) ? new Element('div.eta', {
+						'text': eta_date,
+						'title': 'ETA'
+					}) : null,
 					self.quality = new Element('div.quality', {
 						'events': {
 							'click': function(e){
@@ -250,6 +277,10 @@ var Movie = new Class({
 	getUnprefixedTitle: function(t){
 		if(t.substr(0, 4).toLowerCase() == 'the ')
 			t = t.substr(4) + ', The';
+		else if(t.substr(0, 3).toLowerCase() == 'an ')
+			t = t.substr(3) + ', An';
+		else if(t.substr(0, 2).toLowerCase() == 'a ')
+			t = t.substr(2) + ', A';
 		return t;
 	},
 
@@ -296,6 +327,17 @@ var Movie = new Class({
 		self.el.removeClass(self.view+'_view')
 	},
 
+	getIdentifier: function(){
+		var self = this;
+
+		try {
+			return self.get('identifiers').imdb;
+		}
+		catch (e){ }
+
+		return self.get('imdb');
+	},
+
 	get: function(attr){
 		return this.data[attr] || this.data.info[attr]
 	},

couchpotato/core/media/movie/_base/static/wanted.js

@@ -2,6 +2,7 @@ Page.Wanted = new Class({
 
 	Extends: PageBase,
 
+	order: 10,
 	name: 'wanted',
 	title: 'Gimmy gimmy gimmy!',
 	folder_browser: null,

couchpotato/core/media/movie/charts/__init__.py

@@ -22,11 +22,18 @@ config = [{
                     'description': 'Maximum number of items displayed from each chart.',
                 },
                 {
-                    'name': 'update_interval',
-                    'default': 12,
-                    'type': 'int',
+                    'name': 'hide_wanted',
+                    'default': False,
+                    'type': 'bool',
                     'advanced': True,
-                    'description': '(hours)',
+                    'description': 'Hide the chart movies that are already in your wanted list.',
+                },
+                {
+                    'name': 'hide_library',
+                    'default': False,
+                    'type': 'bool',
+                    'advanced': True,
+                    'description': 'Hide the chart movies that are already in your library.',
                 },
             ],
         },

couchpotato/core/media/movie/charts/main.py

@@ -1,6 +1,5 @@
 import time
 
-from couchpotato import tryInt
 from couchpotato.core.logger import CPLog
 from couchpotato.api import addApiView
 from couchpotato.core.event import addEvent,fireEvent
@@ -13,13 +12,14 @@ log = CPLog(__name__)
 class Charts(Plugin):
 
     update_in_progress = False
+    update_interval = 72 # hours
 
     def __init__(self):
         addApiView('charts.view', self.automationView)
         addEvent('app.load', self.setCrons)
 
     def setCrons(self):
-        fireEvent('schedule.interval', 'charts.update_cache', self.updateViewCache, hours = self.conf('update_interval', default = 12))
+        fireEvent('schedule.interval', 'charts.update_cache', self.updateViewCache, hours = self.update_interval)
 
     def automationView(self, force_update = False, **kwargs):
 
@@ -36,7 +36,6 @@ class Charts(Plugin):
             'charts': charts
         }
 
-
     def updateViewCache(self):
 
         if self.update_in_progress:
@@ -46,10 +45,14 @@ class Charts(Plugin):
             if catched_charts:
                 return catched_charts
 
+        charts = []
         try:
             self.update_in_progress = True
             charts = fireEvent('automation.get_chart_list', merge = True)
-            self.setCache('charts_cached', charts, timeout = 7200 * tryInt(self.conf('update_interval', default = 12)))
+            for chart in charts:
+                chart['hide_wanted'] = self.conf('hide_wanted')
+                chart['hide_library'] = self.conf('hide_library')
+            self.setCache('charts_cached', charts, timeout = self.update_interval * 3600)
         except:
             log.error('Failed refreshing charts')
 

couchpotato/core/media/movie/charts/static/charts.css

@@ -3,15 +3,21 @@
 	margin-bottom: 30px;
 }
 
-.charts > h2 {
-	height: 40px;
-}
+    .charts > h2 {
+        height: 40px;
+    }
 
-.charts .chart {
-	display: inline-block;
-	width: 50%;
-	vertical-align: top;
-}
+    .charts .chart {
+        display: inline-block;
+        width: 50%;
+        vertical-align: top;
+        max-height: 510px;
+        scrollbar-base-color: #4e5969;
+    }
+
+    .charts .chart .media_result.hidden {
+        display: none;
+    }
 
 .charts .refresh {
 	clear:both;
@@ -25,30 +31,30 @@
 		text-align:center;
 	}
 
-	.charts .refresh a {
-		text-align: center;
-		padding: 0;
-		display: none;
-		width: 30px;
-		height: 30px;
-		position: absolute;
-		right: 10px;
-		top: -40px;
-		opacity: .7;
-	}
+        .charts .refresh a {
+            text-align: center;
+            padding: 0;
+            display: none;
+            width: 30px;
+            height: 30px;
+            position: absolute;
+            right: 10px;
+            top: -40px;
+            opacity: .7;
+        }
 
-	.charts .refresh a:hover {
-		opacity: 1;
-	}
+            .charts .refresh a:hover {
+                opacity: 1;
+            }
 
-.charts p.no_charts_enabled {
-	padding: 0.7em 1em;
-	display: none;
-}
+    .charts p.no_charts_enabled {
+         padding: 0.7em 1em;
+         display: none;
+     }
 
-.charts .chart h3 a {
-	color: #fff;
-}
+    .charts .chart h3 a {
+        color: #fff;
+    }
 
 
 .charts .chart .media_result {
@@ -137,7 +143,6 @@
 		padding: 0 3px 10px 0;
 	}
 		.charts .chart .media_result .data:before {
-			bottom: 0;
 			content: '';
 			display: block;
 			height: 10px;

couchpotato/core/media/movie/charts/static/charts.js

@@ -2,6 +2,8 @@ var Charts = new Class({
 
 	Implements: [Options, Events],
 
+	shown_once: false,
+
 	initialize: function(options){
 		var self = this;
 		self.setOptions(options);
@@ -22,9 +24,11 @@ var Charts = new Class({
 					'events': {
 						'click': function(e) {
 							e.preventDefault();
-							self.el.getChildren('div.chart').destroy();
+
+							self.el.getElements('.chart').destroy();
 							self.el_refreshing_text.show();
 							self.el_refresh_link.hide();
+
 							self.api_request = Api.request('charts.view', {
 								'data': { 'force_update': 1 },
 								'onComplete': self.fill.bind(self)
@@ -38,17 +42,13 @@ var Charts = new Class({
 			)
 		);
 
-		if( Cookie.read('suggestions_charts_menu_selected') === 'charts')
-			self.el.show();
+		if( Cookie.read('suggestions_charts_menu_selected') === 'charts'){
+			self.show();
+			self.fireEvent.delay(0, self, 'created');
+		}
 		else
 			self.el.hide();
 
-		self.api_request = Api.request('charts.view', {
-			'onComplete': self.fill.bind(self)
-		});
-
-		self.fireEvent.delay(0, self, 'created');
-
 	},
 
 	fill: function(json){
@@ -72,7 +72,7 @@ var Charts = new Class({
 
 			Object.each(json.charts, function(chart){
 
-				var c = new Element('div.chart').grab(
+				var c = new Element('div.chart.tiny_scroll').grab(
 					new Element('h3').grab( new Element('a', {
 						'text': chart.name,
 						'href': chart.url
@@ -89,7 +89,7 @@ var Charts = new Class({
 						}
 					});
 
-					var in_database_class = movie.in_wanted ? 'chart_in_wanted' : (movie.in_library ? 'chart_in_library' : ''),
+					var in_database_class = (chart.hide_wanted && movie.in_wanted) ? 'hidden' : (movie.in_wanted ? 'chart_in_wanted' : ((chart.hide_library && movie.in_library) ? 'hidden': (movie.in_library ? 'chart_in_library' : ''))),
 						in_database_title = movie.in_wanted ? 'Movie in wanted list' : (movie.in_library ? 'Movie in library' : '');
 
 					m.el
@@ -155,6 +155,24 @@ var Charts = new Class({
 
 	},
 
+	show: function(){
+		var self = this;
+
+		self.el.show();
+
+		if(!self.shown_once){
+			self.api_request = Api.request('charts.view', {
+				'onComplete': self.fill.bind(self)
+			});
+
+			self.shown_once = true;
+		}
+	},
+
+	hide: function(){
+		this.el.hide();
+	},
+
 	afterAdded: function(m){
 
 		$(m).getElement('div.chart_number')

couchpotato/core/media/movie/library.py

@@ -1,4 +1,5 @@
 from couchpotato.core.event import addEvent
+from couchpotato.core.helpers.variable import getTitle
 from couchpotato.core.logger import CPLog
 from couchpotato.core.media._base.library.base import LibraryBase
 
@@ -17,7 +18,9 @@ class MovieLibraryPlugin(LibraryBase):
         if media.get('type') != 'movie':
             return
 
+        default_title = getTitle(media)
         titles = media['info'].get('titles', [])
+        titles.insert(0, default_title)
 
         # Add year identifier to titles
         if include_year:

couchpotato/core/media/movie/providers/automation/bluray.py

@@ -1,9 +1,13 @@
+import traceback
+
 from bs4 import BeautifulSoup
+from couchpotato import fireEvent
 from couchpotato.core.helpers.rss import RSS
 from couchpotato.core.helpers.variable import tryInt
 from couchpotato.core.logger import CPLog
 from couchpotato.core.media.movie.providers.automation.base import Automation
 
+
 log = CPLog(__name__)
 
 autoload = 'Bluray'
@@ -33,27 +37,49 @@ class Bluray(Automation, RSS):
 
                 try:
                     # Stop if the release year is before the minimal year
-                    page_year = soup.body.find_all('center')[3].table.tr.find_all('td', recursive = False)[3].h3.get_text().split(', ')[1]
-                    if tryInt(page_year) < self.getMinimal('year'):
+                    brk = False
+                    h3s = soup.body.find_all('h3')
+                    for h3 in h3s:
+                        if h3.parent.name != 'a':
+
+                            try:
+                                page_year = tryInt(h3.get_text()[-4:])
+                                if page_year > 0 and page_year < self.getMinimal('year'):
+                                    brk = True
+                            except:
+                                log.error('Failed determining page year: %s', traceback.format_exc())
+                                brk = True
+                                break
+
+                    if brk:
                         break
 
-                    for table in soup.body.find_all('center')[3].table.tr.find_all('td', recursive = False)[3].find_all('table')[1:20]:
-                        name = table.h3.get_text().lower().split('blu-ray')[0].strip()
-                        year = table.small.get_text().split('|')[1].strip()
+                    for h3 in h3s:
+                        try:
+                            if h3.parent.name == 'a':
+                                name = h3.get_text().lower().split('blu-ray')[0].strip()
 
-                        if not name.find('/') == -1:  # make sure it is not a double movie release
-                            continue
+                                if not name.find('/') == -1:  # make sure it is not a double movie release
+                                    continue
 
-                        if tryInt(year) < self.getMinimal('year'):
-                            continue
+                                if not h3.parent.parent.small:  # ignore non-movie tables
+                                    continue
 
-                        imdb = self.search(name, year)
+                                year = h3.parent.parent.small.get_text().split('|')[1].strip()
 
-                        if imdb:
-                            if self.isMinimalMovie(imdb):
-                                movies.append(imdb['imdb'])
+                                if tryInt(year) < self.getMinimal('year'):
+                                    continue
+
+                                imdb = self.search(name, year)
+
+                                if imdb:
+                                    if self.isMinimalMovie(imdb):
+                                        movies.append(imdb['imdb'])
+                        except:
+                            log.debug('Error parsing movie html: %s', traceback.format_exc())
+                            break
                 except:
-                    log.debug('Error loading page: %s', page)
+                    log.debug('Error loading page %s: %s', (page, traceback.format_exc()))
                     break
 
             self.conf('backlog', value = False)
@@ -82,6 +108,7 @@ class Bluray(Automation, RSS):
     def getChartList(self):
         # Nearly identical to 'getIMDBids', but we don't care about minimalMovie and return all movie data (not just id)
         movie_list = {'name': 'Blu-ray.com - New Releases', 'url': self.display_url, 'order': self.chart_order, 'list': []}
+        movie_ids = []
         max_items = int(self.conf('max_items', section='charts', default=5))
         rss_movies = self.getRSSData(self.rss_url)
 
@@ -95,6 +122,15 @@ class Bluray(Automation, RSS):
             movie = self.search(name, year)
 
             if movie:
+
+                if movie.get('imdb') in movie_ids:
+                    continue
+
+                is_movie = fireEvent('movie.is_movie', identifier = movie.get('imdb'), single = True)
+                if not is_movie:
+                    continue
+
+                movie_ids.append(movie.get('imdb'))
                 movie_list['list'].append( movie )
                 if len(movie_list['list']) >= max_items:
                     break
@@ -123,7 +159,7 @@ config = [{
                 {
                     'name': 'backlog',
                     'advanced': True,
-                    'description': 'Parses the history until the minimum movie year is reached. (Will be disabled once it has completed)',
+                    'description': ('Parses the history until the minimum movie year is reached. (Takes a while)', 'Will be disabled once it has completed'),
                     'default': False,
                     'type': 'bool',
                 },

couchpotato/core/media/movie/providers/automation/imdb.py

@@ -3,6 +3,7 @@ import re
 
 from bs4 import BeautifulSoup
 from couchpotato import fireEvent
+from couchpotato.core.helpers.encoding import ss
 from couchpotato.core.helpers.rss import RSS
 from couchpotato.core.helpers.variable import getImdb, splitString, tryInt
 from couchpotato.core.logger import CPLog
@@ -28,6 +29,39 @@ class IMDBBase(Automation, RSS):
     def getInfo(self, imdb_id):
         return fireEvent('movie.info', identifier = imdb_id, extended = False, merge = True)
 
+    def getFromURL(self, url):
+        log.debug('Getting IMDBs from: %s', url)
+        html = self.getHTMLData(url)
+
+        try:
+            split = splitString(html, split_on = "<div class=\"list compact\">")[1]
+            html = splitString(split, split_on = "<div class=\"pages\">")[0]
+        except:
+            try:
+                split = splitString(html, split_on = "<div id=\"main\">")
+
+                if len(split) < 2:
+                    log.error('Failed parsing IMDB page "%s", unexpected html.', url)
+                    return []
+
+                html = BeautifulSoup(split[1])
+                for x in ['list compact', 'lister', 'list detail sub-list']:
+                    html2 = html.find('div', attrs = {
+                        'class': x
+                    })
+
+                    if html2:
+                        html = html2.contents
+                        html = ''.join([str(x) for x in html])
+                        break
+            except:
+                log.error('Failed parsing IMDB page "%s": %s', (url, traceback.format_exc()))
+
+        html = ss(html)
+        imdbs = getImdb(html, multiple = True) if html else []
+
+        return imdbs
+
 
 class IMDBWatchlist(IMDBBase):
 
@@ -65,16 +99,7 @@ class IMDBWatchlist(IMDBBase):
                 try:
 
                     w_url = '%s&start=%s' % (watchlist_url, start)
-                    log.debug('Started IMDB watchlists: %s', w_url)
-                    html = self.getHTMLData(w_url)
-
-                    try:
-                        split = splitString(html, split_on="<div class=\"list compact\">")[1]
-                        html = splitString(split, split_on="<div class=\"pages\">")[0]
-                    except:
-                        pass
-
-                    imdbs = getImdb(html, multiple = True) if html else []
+                    imdbs = self.getFromURL(w_url)
 
                     for imdb in imdbs:
                         if imdb not in movies:
@@ -85,13 +110,14 @@ class IMDBWatchlist(IMDBBase):
 
                     log.debug('Found %s movies on %s', (len(imdbs), w_url))
 
-                    if len(imdbs) < 250:
+                    if len(imdbs) < 225:
                         break
 
-                    start += 250
+                    start = len(movies)
 
                 except:
                     log.error('Failed loading IMDB watchlist: %s %s', (watchlist_url, traceback.format_exc()))
+                    break
 
         return movies
 
@@ -100,95 +126,88 @@ class IMDBAutomation(IMDBBase):
 
     enabled_option = 'automation_providers_enabled'
 
-    chart_urls = {
-        'theater': 'http://www.imdb.com/movies-in-theaters/',
-        'top250': 'http://www.imdb.com/chart/top',
-        'boxoffice': 'http://www.imdb.com/chart/',
+    charts = {
+        'theater': {
+            'order': 1,
+            'name': 'IMDB - Movies in Theaters',
+            'url': 'http://www.imdb.com/movies-in-theaters/',
+        },
+        'boxoffice': {
+            'order': 2,
+            'name': 'IMDB - Box Office',
+            'url': 'http://www.imdb.com/boxoffice/',
+        },
+        'rentals': {
+            'order': 3,
+            'name': 'IMDB - Top DVD rentals',
+            'url': 'http://www.imdb.com/boxoffice/rentals',
+            'type': 'json',
+        },
+        'top250': {
+            'order': 4,
+            'name': 'IMDB - Top 250 Movies',
+            'url': 'http://www.imdb.com/chart/top',
+        },
     }
-    chart_names = {
-        'theater': 'IMDB - Movies in Theaters',
-        'top250': 'IMDB - Top 250 Movies',
-        'boxoffice': 'IMDB - Box Office',
-    }
-    chart_order = {
-        'theater': 2,
-        'top250': 4,
-        'boxoffice': 3,
-    }
-
-    first_table = ['boxoffice']
 
     def getIMDBids(self):
 
         movies = []
 
-        for url in self.chart_urls:
-            if self.conf('automation_charts_%s' % url):
-                data = self.getHTMLData(self.chart_urls[url])
-                if data:
-                    html = BeautifulSoup(data)
+        for name in self.charts:
+            chart = self.charts[name]
+            url = chart.get('url')
 
-                    try:
-                        result_div = html.find('div', attrs = {'id': 'main'})
+            if self.conf('automation_charts_%s' % name):
+                imdb_ids = self.getFromURL(url)
 
-                        try:
-                            if url in self.first_table:
-                                table = result_div.find('table')
-                                result_div = table if table else result_div
-                        except:
-                            pass
+                try:
+                    for imdb_id in imdb_ids:
+                        info = self.getInfo(imdb_id)
+                        if info and self.isMinimalMovie(info):
+                            movies.append(imdb_id)
 
-                        imdb_ids = getImdb(str(result_div), multiple = True)
+                        if self.shuttingDown():
+                            break
 
-                        for imdb_id in imdb_ids:
-                            info = self.getInfo(imdb_id)
-                            if info and self.isMinimalMovie(info):
-                                movies.append(imdb_id)
-
-                            if self.shuttingDown():
-                                break
-
-                    except:
-                        log.error('Failed loading IMDB chart results from %s: %s', (url, traceback.format_exc()))
+                except:
+                    log.error('Failed loading IMDB chart results from %s: %s', (url, traceback.format_exc()))
 
         return movies
 
-
     def getChartList(self):
+
         # Nearly identical to 'getIMDBids', but we don't care about minimalMovie and return all movie data (not just id)
         movie_lists = []
-        max_items = int(self.conf('max_items', section='charts', default=5))
+        max_items = int(self.conf('max_items', section = 'charts', default=5))
 
-        for url in self.chart_urls:
-            if self.conf('chart_display_%s' % url):
-                movie_list = {'name': self.chart_names[url], 'url': self.chart_urls[url], 'order': self.chart_order[url], 'list': []}
-                data = self.getHTMLData(self.chart_urls[url])
-                if data:
-                    html = BeautifulSoup(data)
+        for name in self.charts:
+            chart = self.charts[name].copy()
+            url = chart.get('url')
 
-                    try:
-                        result_div = html.find('div', attrs = {'id': 'main'})
+            if self.conf('chart_display_%s' % name):
 
-                        try:
-                            if url in self.first_table:
-                                table = result_div.find('table')
-                                result_div = table if table else result_div
-                        except:
-                            pass
+                chart['list'] = []
 
-                        imdb_ids = getImdb(str(result_div), multiple = True)
+                imdb_ids = self.getFromURL(url)
 
-                        for imdb_id in imdb_ids[0:max_items]:
-                            info = self.getInfo(imdb_id)
-                            movie_list['list'].append(info)
+                try:
+                    for imdb_id in imdb_ids[0:max_items]:
 
-                            if self.shuttingDown():
-                                break
-                    except:
-                        log.error('Failed loading IMDB chart results from %s: %s', (url, traceback.format_exc()))
+                        is_movie = fireEvent('movie.is_movie', identifier = imdb_id, single = True)
+                        if not is_movie:
+                            continue
 
-                    if movie_list['list']:
-                        movie_lists.append(movie_list)
+                        info = self.getInfo(imdb_id)
+                        chart['list'].append(info)
+
+                        if self.shuttingDown():
+                            break
+                except:
+                    log.error('Failed loading IMDB chart results from %s: %s', (url, traceback.format_exc()))
+
+                if chart['list']:
+                    movie_lists.append(chart)
 
 
         return movie_lists
@@ -240,12 +259,19 @@ config = [{
                     'description': 'New Movies <a href="http://www.imdb.com/movies-in-theaters/">In-Theaters</a> chart',
                     'default': True,
                 },
+                {
+                    'name': 'automation_charts_rentals',
+                    'type': 'bool',
+                    'label': 'DVD Rentals',
+                    'description': 'Top DVD <a href="http://www.imdb.com/boxoffice/rentals">rentals</a> chart',
+                    'default': True,
+                },
                 {
                     'name': 'automation_charts_top250',
                     'type': 'bool',
                     'label': 'TOP 250',
                     'description': 'IMDB <a href="http://www.imdb.com/chart/top/">TOP 250</a> chart',
-                    'default': True,
+                    'default': False,
                 },
                 {
                     'name': 'automation_charts_boxoffice',
@@ -282,6 +308,13 @@ config = [{
                     'description': 'IMDB <a href="http://www.imdb.com/chart/top/">TOP 250</a> chart',
                     'default': False,
                 },
+                {
+                    'name': 'chart_display_rentals',
+                    'type': 'bool',
+                    'label': 'DVD Rentals',
+                    'description': 'Top DVD <a href="http://www.imdb.com/boxoffice/rentals">rentals</a> chart',
+                    'default': True,
+                },
                 {
                     'name': 'chart_display_boxoffice',
                     'type': 'bool',

couchpotato/core/media/movie/providers/automation/moviemeter.py

@@ -21,11 +21,15 @@ class Moviemeter(Automation, RSS):
 
         for movie in rss_movies:
 
-            name_year = fireEvent('scanner.name_year', self.getTextElement(movie, 'title'), single = True)
-            imdb = self.search(name_year.get('name'), name_year.get('year'))
+            title = self.getTextElement(movie, 'title')
+            name_year = fireEvent('scanner.name_year', title, single = True)
+            if name_year.get('name') and name_year.get('year'):
+                imdb = self.search(name_year.get('name'), name_year.get('year'))
 
-            if imdb and self.isMinimalMovie(imdb):
-                movies.append(imdb['imdb'])
+                if imdb and self.isMinimalMovie(imdb):
+                    movies.append(imdb['imdb'])
+            else:
+                log.error('Failed getting name and year from: %s', title)
 
         return movies
 

couchpotato/core/media/movie/providers/automation/popularmovies.py

@@ -0,0 +1,47 @@
+from couchpotato import fireEvent
+from couchpotato.core.logger import CPLog
+from couchpotato.core.media.movie.providers.automation.base import Automation
+
+log = CPLog(__name__)
+
+autoload = 'PopularMovies'
+
+
+class PopularMovies(Automation):
+
+    interval = 1800
+    url = 'https://s3.amazonaws.com/popular-movies/movies.json'
+
+    def getIMDBids(self):
+
+        movies = []
+        retrieved_movies = self.getJsonData(self.url)
+
+        for movie in retrieved_movies.get('movies'):
+            imdb_id = movie.get('imdb_id')
+            info = fireEvent('movie.info', identifier = imdb_id, extended = False, merge = True)
+            if self.isMinimalMovie(info):
+                movies.append(imdb_id)
+
+        return movies
+
+
+config = [{
+    'name': 'popularmovies',
+    'groups': [
+        {
+            'tab': 'automation',
+            'list': 'automation_providers',
+            'name': 'popularmovies_automation',
+            'label': 'Popular Movies',
+            'description': 'Imports the <a href="http://movies.stevenlu.com/">top titles of movies that have been in theaters</a>. Script provided by <a href="https://github.com/sjlu/popular-movies">Steven Lu</a>',
+            'options': [
+                {
+                    'name': 'automation_enabled',
+                    'default': False,
+                    'type': 'enabler',
+                },
+            ],
+        },
+    ],
+}]

couchpotato/core/media/movie/providers/info/_modifier.py

@@ -1,9 +1,9 @@
 import copy
 import traceback
 
-from CodernityDB.database import RecordNotFound
 from couchpotato import get_db
 from couchpotato.core.event import addEvent, fireEvent
+from couchpotato.core.helpers.database import RecordNotFound
 from couchpotato.core.helpers.variable import mergeDicts, randomString
 from couchpotato.core.logger import CPLog
 from couchpotato.core.plugins.base import Plugin
@@ -26,7 +26,14 @@ class MovieResultModifier(Plugin):
             'backdrop': [],
             'poster_original': [],
             'backdrop_original': [],
-            'actors': {}
+            'actors': {},
+            'landscape': [],
+            'logo': [],
+            'clear_art': [],
+            'disc_art': [],
+            'banner': [],
+            'extra_thumbs': [],
+            'extra_fanart': []
         },
         'runtime': 0,
         'plot': '',

couchpotato/core/media/movie/providers/info/couchpotatoapi.py

@@ -2,7 +2,7 @@ import base64
 import time
 
 from couchpotato.core.event import addEvent, fireEvent
-from couchpotato.core.helpers.encoding import tryUrlencode
+from couchpotato.core.helpers.encoding import tryUrlencode, ss
 from couchpotato.core.logger import CPLog
 from couchpotato.core.media.movie.providers.base import MovieProvider
 from couchpotato.environment import Env
@@ -29,7 +29,7 @@ class CouchPotatoApi(MovieProvider):
     api_version = 1
 
     def __init__(self):
-        addEvent('movie.info', self.getInfo, priority = 1)
+        addEvent('movie.info', self.getInfo, priority = 2)
         addEvent('movie.info.release_date', self.getReleaseDate)
 
         addEvent('info.search', self.search, priority = 1)
@@ -66,7 +66,7 @@ class CouchPotatoApi(MovieProvider):
         if not name:
             return
 
-        name_enc = base64.b64encode(name)
+        name_enc = base64.b64encode(ss(name))
         return self.getJsonData(self.urls['validate'] % name_enc, headers = self.getRequestHeaders())
 
     def isMovie(self, identifier = None):

couchpotato/core/media/movie/providers/info/fanarttv.py

@@ -0,0 +1,130 @@
+import traceback
+
+from couchpotato import tryInt
+from couchpotato.core.event import addEvent
+from couchpotato.core.logger import CPLog
+from couchpotato.core.media.movie.providers.base import MovieProvider
+
+
+log = CPLog(__name__)
+
+autoload = 'FanartTV'
+
+
+class FanartTV(MovieProvider):
+
+    urls = {
+        'api': 'http://webservice.fanart.tv/v3/movies/%s?api_key=b28b14e9be662e027cfbc7c3dd600405'
+    }
+
+    MAX_EXTRAFANART = 20
+    http_time_between_calls = 0
+
+    def __init__(self):
+        addEvent('movie.info', self.getArt, priority = 1)
+
+    def getArt(self, identifier = None, extended = True, **kwargs):
+
+        if not identifier or not extended:
+            return {}
+
+        images = {}
+
+        try:
+            url = self.urls['api'] % identifier
+            fanart_data = self.getJsonData(url)
+
+            if fanart_data:
+                log.debug('Found images for %s', fanart_data.get('name'))
+                images = self._parseMovie(fanart_data)
+
+        except:
+            log.error('Failed getting extra art for %s: %s',
+                      (identifier, traceback.format_exc()))
+            return {}
+
+        return {
+            'images': images
+        }
+
+    def _parseMovie(self, movie):
+        images = {
+            'landscape': self._getMultImages(movie.get('moviethumb', []), 1),
+            'logo': [],
+            'disc_art': self._getMultImages(self._trimDiscs(movie.get('moviedisc', [])), 1),
+            'clear_art': self._getMultImages(movie.get('hdmovieart', []), 1),
+            'banner': self._getMultImages(movie.get('moviebanner', []), 1),
+            'extra_fanart': [],
+        }
+
+        if len(images['clear_art']) == 0:
+            images['clear_art'] = self._getMultImages(movie.get('movieart', []), 1)
+
+        images['logo'] = self._getMultImages(movie.get('hdmovielogo', []), 1)
+        if len(images['logo']) == 0:
+            images['logo'] = self._getMultImages(movie.get('movielogo', []), 1)
+
+        fanarts = self._getMultImages(movie.get('moviebackground', []), self.MAX_EXTRAFANART + 1)
+
+        if fanarts:
+            images['backdrop_original'] = [fanarts[0]]
+            images['extra_fanart'] = fanarts[1:]
+
+        return images
+
+    def _trimDiscs(self, disc_images):
+        """
+        Return a subset of discImages. Only bluray disc images will be returned.
+        """
+
+        trimmed = []
+        for disc in disc_images:
+            if disc.get('disc_type') == 'bluray':
+                trimmed.append(disc)
+
+        if len(trimmed) == 0:
+            return disc_images
+
+        return trimmed
+
+    def _getImage(self, images):
+        image_url = None
+        highscore = -1
+        for image in images:
+            if tryInt(image.get('likes')) > highscore:
+                highscore = tryInt(image.get('likes'))
+                image_url = image.get('url') or image.get('href')
+
+        return image_url
+
+    def _getMultImages(self, images, n):
+        """
+        Chooses the best n images and returns them as a list.
+        If n<0, all images will be returned.
+        """
+        image_urls = []
+        pool = []
+        for image in images:
+            if image.get('lang') == 'en':
+                pool.append(image)
+        orig_pool_size = len(pool)
+
+        while len(pool) > 0 and (n < 0 or orig_pool_size - len(pool) < n):
+            best = None
+            highscore = -1
+            for image in pool:
+                if tryInt(image.get('likes')) > highscore:
+                    highscore = tryInt(image.get('likes'))
+                    best = image
+            url = best.get('url') or best.get('href')
+            if url:
+                image_urls.append(url)
+            pool.remove(best)
+
+        return image_urls
+
+    def isDisabled(self):
+        if self.conf('api_key') == '':
+            log.error('No API key provided.')
+            return True
+        return False

couchpotato/core/media/movie/providers/info/omdbapi.py

@@ -7,6 +7,7 @@ from couchpotato.core.helpers.encoding import tryUrlencode
 from couchpotato.core.helpers.variable import tryInt, tryFloat, splitString
 from couchpotato.core.logger import CPLog
 from couchpotato.core.media.movie.providers.base import MovieProvider
+import six
 
 
 log = CPLog(__name__)
@@ -72,7 +73,7 @@ class OMDBAPI(MovieProvider):
         try:
 
             try:
-                if isinstance(movie, (str, unicode)):
+                if isinstance(movie, six.string_types):
                     movie = json.loads(movie)
             except ValueError:
                 log.info('No proper json to decode')

couchpotato/core/media/movie/providers/info/themoviedb.py

@@ -1,11 +1,11 @@
 import traceback
+import time
 
-from couchpotato.core.event import addEvent
-from couchpotato.core.helpers.encoding import simplifyString, toUnicode, ss
+from couchpotato.core.event import addEvent, fireEvent
+from couchpotato.core.helpers.encoding import simplifyString, toUnicode, ss, tryUrlencode
 from couchpotato.core.helpers.variable import tryInt
 from couchpotato.core.logger import CPLog
 from couchpotato.core.media.movie.providers.base import MovieProvider
-import tmdb3
 
 log = CPLog(__name__)
 
@@ -14,13 +14,23 @@ autoload = 'TheMovieDb'
 
 class TheMovieDb(MovieProvider):
 
-    def __init__(self):
-        addEvent('movie.info', self.getInfo, priority = 2)
-        addEvent('movie.info_by_tmdb', self.getInfo)
+    http_time_between_calls = .3
 
-        # Configure TMDB settings
-        tmdb3.set_key(self.conf('api_key'))
-        tmdb3.set_cache('null')
+    configuration = {
+        'images': {
+            'secure_base_url': 'https://image.tmdb.org/t/p/',
+        },
+    }
+
+    def __init__(self):
+        addEvent('movie.info', self.getInfo, priority = 3)
+        addEvent('movie.info_by_tmdb', self.getInfo)
+        addEvent('app.load', self.config)
+
+    def config(self):
+        configuration = self.request('configuration')
+        if configuration:
+            self.configuration = configuration
 
     def search(self, q, limit = 12):
         """ Find movie by name """
@@ -30,14 +40,19 @@ class TheMovieDb(MovieProvider):
 
         search_string = simplifyString(q)
         cache_key = 'tmdb.cache.%s.%s' % (search_string, limit)
-        results = self.getCache(cache_key)
+        results = None #self.getCache(cache_key)
 
         if not results:
             log.debug('Searching for movie: %s', q)
 
             raw = None
             try:
-                raw = tmdb3.searchMovie(search_string)
+
+                #name_year = fireEvent('scanner.name_year', q, single = True)
+
+                raw = self.request('search/movie', {
+                    'query': q
+                }, return_key = 'results')
             except:
                 log.error('Failed searching TMDB for "%s": %s', (search_string, traceback.format_exc()))
 
@@ -68,92 +83,92 @@ class TheMovieDb(MovieProvider):
         if not identifier:
             return {}
 
-        cache_key = 'tmdb.cache.%s%s' % (identifier, '.ex' if extended else '')
-        result = self.getCache(cache_key)
-
-        if not result:
-            try:
-                log.debug('Getting info: %s', cache_key)
-                # noinspection PyArgumentList
-                movie = tmdb3.Movie(identifier)
-                try: exists = movie.title is not None
-                except: exists = False
-
-                if exists:
-                    result = self.parseMovie(movie, extended = extended)
-                    self.setCache(cache_key, result)
-                else:
-                    result = {}
-            except:
-                log.error('Failed getting info for %s: %s', (identifier, traceback.format_exc()))
+        result = self.parseMovie({
+            'id': identifier
+        }, extended = extended)
 
         return result
 
     def parseMovie(self, movie, extended = True):
 
-        cache_key = 'tmdb.cache.%s%s' % (movie.id, '.ex' if extended else '')
-        movie_data = self.getCache(cache_key)
+        cache_key = 'tmdb.cache.%s%s' % (movie.get('id'), '.ex' if extended else '')
+        movie_data = None #self.getCache(cache_key)
 
         if not movie_data:
 
+            # Full data
+            movie = self.request('movie/%s' % movie.get('id'))
+
             # Images
-            poster = self.getImage(movie, type = 'poster', size = 'poster')
+            poster = self.getImage(movie, type = 'poster', size = 'w154')
             poster_original = self.getImage(movie, type = 'poster', size = 'original')
             backdrop_original = self.getImage(movie, type = 'backdrop', size = 'original')
+            extra_thumbs = self.getMultImages(movie, type = 'backdrops', size = 'original')
 
             images = {
                 'poster': [poster] if poster else [],
                 #'backdrop': [backdrop] if backdrop else [],
                 'poster_original': [poster_original] if poster_original else [],
                 'backdrop_original': [backdrop_original] if backdrop_original else [],
-                'actors': {}
+                'actors': {},
+                'extra_thumbs': extra_thumbs
             }
 
             # Genres
             try:
-                genres = [genre.name for genre in movie.genres]
+                genres = [genre.get('name') for genre in movie.get('genres', [])]
             except:
                 genres = []
 
             # 1900 is the same as None
-            year = str(movie.releasedate or '')[:4]
-            if not movie.releasedate or year == '1900' or year.lower() == 'none':
+            year = str(movie.get('release_date') or '')[:4]
+            if not movie.get('release_date') or year == '1900' or year.lower() == 'none':
                 year = None
 
             # Gather actors data
             actors = {}
             if extended:
-                for cast_item in movie.cast:
+
+                # Full data
+                cast = self.request('movie/%s/casts' % movie.get('id'), return_key = 'cast')
+
+                for cast_item in cast:
                     try:
-                        actors[toUnicode(cast_item.name)] = toUnicode(cast_item.character)
-                        images['actors'][toUnicode(cast_item.name)] = self.getImage(cast_item, type = 'profile', size = 'original')
+                        actors[toUnicode(cast_item.get('name'))] = toUnicode(cast_item.get('character'))
+                        images['actors'][toUnicode(cast_item.get('name'))] = self.getImage(cast_item, type = 'profile', size = 'original')
                     except:
                         log.debug('Error getting cast info for %s: %s', (cast_item, traceback.format_exc()))
 
             movie_data = {
                 'type': 'movie',
                 'via_tmdb': True,
-                'tmdb_id': movie.id,
-                'titles': [toUnicode(movie.title)],
-                'original_title': movie.originaltitle,
+                'tmdb_id': movie.get('id'),
+                'titles': [toUnicode(movie.get('title'))],
+                'original_title': movie.get('original_title'),
                 'images': images,
-                'imdb': movie.imdb,
-                'runtime': movie.runtime,
-                'released': str(movie.releasedate),
+                'imdb': movie.get('imdb_id'),
+                'runtime': movie.get('runtime'),
+                'released': str(movie.get('release_date')),
                 'year': tryInt(year, None),
-                'plot': movie.overview,
+                'plot': movie.get('overview'),
                 'genres': genres,
-                'collection': getattr(movie.collection, 'name', None),
+                'collection': getattr(movie.get('belongs_to_collection'), 'name', None),
                 'actor_roles': actors
             }
 
             movie_data = dict((k, v) for k, v in movie_data.items() if v)
 
             # Add alternative names
+            if movie_data['original_title'] and movie_data['original_title'] not in movie_data['titles']:
+                movie_data['titles'].append(movie_data['original_title'])
+
             if extended:
-                movie_data['titles'].append(movie.originaltitle)
-                for alt in movie.alternate_titles:
-                    alt_name = alt.title
+
+                # Full data
+                alternate_titles = self.request('movie/%s/alternative_titles' % movie.get('id'), return_key = 'titles')
+
+                for alt in alternate_titles:
+                    alt_name = alt.get('title')
                     if alt_name and alt_name not in movie_data['titles'] and alt_name.lower() != 'none' and alt_name is not None:
                         movie_data['titles'].append(alt_name)
 
@@ -166,12 +181,38 @@ class TheMovieDb(MovieProvider):
 
         image_url = ''
         try:
-            image_url = getattr(movie, type).geturl(size = size)
+            path = movie.get('%s_path' % type)
+            image_url = '%s%s%s' % (self.configuration['images']['secure_base_url'], size, path)
         except:
             log.debug('Failed getting %s.%s for "%s"', (type, size, ss(str(movie))))
 
         return image_url
 
+    def getMultImages(self, movie, type = 'backdrops', size = 'original'):
+
+        image_urls = []
+        try:
+
+            # Full data
+            images = self.request('movie/%s/images' % movie.get('id'), return_key = type)
+            for image in images[1:5]:
+                image_urls.append(self.getImage(image, 'file', size))
+
+        except:
+            log.debug('Failed getting %s.%s for "%s"', (type, size, ss(str(movie))))
+
+        return image_urls
+
+    def request(self, call = '', params = {}, return_key = None):
+        params = tryUrlencode(params)
+        url = 'http://api.themoviedb.org/3/%s?api_key=%s%s' % (call, self.conf('api_key'), '&%s' % params if params else '')
+        data = self.getJsonData(url, cache_timeout = 0)
+
+        if data and return_key and data.get(return_key):
+            data = data.get(return_key)
+
+        return data
+
     def isDisabled(self):
         if self.conf('api_key') == '':
             log.error('No API key provided.')

couchpotato/core/media/movie/providers/metadata/base.py

@@ -4,7 +4,7 @@ import traceback
 
 from couchpotato.core.event import addEvent, fireEvent
 from couchpotato.core.helpers.encoding import sp
-from couchpotato.core.helpers.variable import getIdentifier
+from couchpotato.core.helpers.variable import getIdentifier, underscoreToCamel
 from couchpotato.core.logger import CPLog
 from couchpotato.core.media._base.providers.metadata.base import MetaDataBase
 from couchpotato.environment import Env
@@ -28,7 +28,7 @@ class MovieMetaData(MetaDataBase):
 
         # Update library to get latest info
         try:
-            group['media'] = fireEvent('movie.update_info', group['media'].get('_id'), identifier = getIdentifier(group['media']), extended = True, single = True)
+            group['media'] = fireEvent('movie.update', group['media'].get('_id'), identifier = getIdentifier(group['media']), extended = True, single = True)
         except:
             log.error('Failed to update movie, before creating metadata: %s', traceback.format_exc())
 
@@ -38,75 +38,150 @@ class MovieMetaData(MetaDataBase):
 
         movie_info = group['media'].get('info')
 
-        for file_type in ['nfo', 'thumbnail', 'fanart']:
+        for file_type in ['nfo']:
             try:
-                # Get file path
-                name = getattr(self, 'get' + file_type.capitalize() + 'Name')(meta_name, root)
+                self._createType(meta_name, root, movie_info, group, file_type, 0)
+            except:
+                log.error('Unable to create %s file: %s', ('nfo', traceback.format_exc()))
 
-                if name and (self.conf('meta_' + file_type) or self.conf('meta_' + file_type) is None):
-
-                    # Get file content
-                    content = getattr(self, 'get' + file_type.capitalize())(movie_info = movie_info, data = group)
-                    if content:
-                        log.debug('Creating %s file: %s', (file_type, name))
-                        if os.path.isfile(content):
-                            content = sp(content)
-                            name = sp(name)
-
-                            shutil.copy2(content, name)
-                            shutil.copyfile(content, name)
-
-                            # Try and copy stats seperately
-                            try: shutil.copystat(content, name)
-                            except: pass
-                        else:
-                            self.createFile(name, content)
-                            group['renamed_files'].append(name)
-
-                        try:
-                            os.chmod(sp(name), Env.getPermission('file'))
-                        except:
-                            log.debug('Failed setting permissions for %s: %s', (name, traceback.format_exc()))
+        for file_type in ['thumbnail', 'fanart', 'banner', 'disc_art', 'logo', 'clear_art', 'landscape', 'extra_thumbs', 'extra_fanart']:
+            try:
+                if file_type == 'thumbnail':
+                    num_images = len(movie_info['images']['poster_original'])
+                elif file_type == 'fanart':
+                    num_images = len(movie_info['images']['backdrop_original'])
+                else:
+                    num_images = len(movie_info['images'][file_type])
 
+                for i in range(num_images):
+                    self._createType(meta_name, root, movie_info, group, file_type, i)
             except:
                 log.error('Unable to create %s file: %s', (file_type, traceback.format_exc()))
 
+    def _createType(self, meta_name, root, movie_info, group, file_type, i):  # Get file path
+        camelcase_method = underscoreToCamel(file_type.capitalize())
+        name = getattr(self, 'get' + camelcase_method + 'Name')(meta_name, root, i)
+
+        if name and (self.conf('meta_' + file_type) or self.conf('meta_' + file_type) is None):
+
+            # Get file content
+            content = getattr(self, 'get' + camelcase_method)(movie_info = movie_info, data = group, i = i)
+            if content:
+                log.debug('Creating %s file: %s', (file_type, name))
+                if os.path.isfile(content):
+                    content = sp(content)
+                    name = sp(name)
+
+                    if not os.path.exists(os.path.dirname(name)):
+                        os.makedirs(os.path.dirname(name))
+
+                    shutil.copy2(content, name)
+                    shutil.copyfile(content, name)
+
+                    # Try and copy stats seperately
+                    try: shutil.copystat(content, name)
+                    except: pass
+                else:
+                    self.createFile(name, content)
+                    group['renamed_files'].append(name)
+
+                try:
+                    os.chmod(sp(name), Env.getPermission('file'))
+                except:
+                    log.debug('Failed setting permissions for %s: %s', (name, traceback.format_exc()))
+
     def getRootName(self, data = None):
         if not data: data = {}
         return os.path.join(data['destination_dir'], data['filename'])
 
-    def getFanartName(self, name, root):
+    def getFanartName(self, name, root, i):
         return
 
-    def getThumbnailName(self, name, root):
+    def getThumbnailName(self, name, root, i):
         return
 
-    def getNfoName(self, name, root):
+    def getBannerName(self, name, root, i):
         return
 
-    def getNfo(self, movie_info = None, data = None):
+    def getClearArtName(self, name, root, i):
+        return
+
+    def getLogoName(self, name, root, i):
+        return
+
+    def getDiscArtName(self, name, root, i):
+        return
+
+    def getLandscapeName(self, name, root, i):
+        return
+
+    def getExtraThumbsName(self, name, root, i):
+        return
+
+    def getExtraFanartName(self, name, root, i):
+        return
+
+    def getNfoName(self, name, root, i):
+        return
+
+    def getNfo(self, movie_info = None, data = None, i = 0):
         if not data: data = {}
         if not movie_info: movie_info = {}
 
-    def getThumbnail(self, movie_info = None, data = None, wanted_file_type = 'poster_original'):
+    def getThumbnail(self, movie_info = None, data = None, wanted_file_type = 'poster_original', i = 0):
         if not data: data = {}
         if not movie_info: movie_info = {}
 
         # See if it is in current files
         files = data['media'].get('files')
         if files.get('image_' + wanted_file_type):
-            if os.path.isfile(files['image_' + wanted_file_type][0]):
-                return files['image_' + wanted_file_type][0]
+            if os.path.isfile(files['image_' + wanted_file_type][i]):
+                return files['image_' + wanted_file_type][i]
 
         # Download using existing info
         try:
             images = movie_info['images'][wanted_file_type]
-            file_path = fireEvent('file.download', url = images[0], single = True)
+            file_path = fireEvent('file.download', url = images[i], single = True)
             return file_path
         except:
             pass
 
-    def getFanart(self, movie_info = None, data = None):
+    def getFanart(self, movie_info = None, data = None, i = 0):
         if not data: data = {}
         if not movie_info: movie_info = {}
-        return self.getThumbnail(movie_info = movie_info, data = data, wanted_file_type = 'backdrop_original')
+        return self.getThumbnail(movie_info = movie_info, data = data, wanted_file_type = 'backdrop_original', i = i)
+
+    def getBanner(self, movie_info = None, data = None, i = 0):
+        if not data: data = {}
+        if not movie_info: movie_info = {}
+        return self.getThumbnail(movie_info = movie_info, data = data, wanted_file_type = 'banner', i = i)
+
+    def getClearArt(self, movie_info = None, data = None, i = 0):
+        if not data: data = {}
+        if not movie_info: movie_info = {}
+        return self.getThumbnail(movie_info = movie_info, data = data, wanted_file_type = 'clear_art', i = i)
+
+    def getLogo(self, movie_info = None, data = None, i = 0):
+        if not data: data = {}
+        if not movie_info: movie_info = {}
+        return self.getThumbnail(movie_info = movie_info, data = data, wanted_file_type = 'logo', i = i)
+
+    def getDiscArt(self, movie_info = None, data = None, i = 0):
+        if not data: data = {}
+        if not movie_info: movie_info = {}
+        return self.getThumbnail(movie_info = movie_info, data = data, wanted_file_type = 'disc_art', i = i)
+
+    def getLandscape(self, movie_info = None, data = None, i = 0):
+        if not data: data = {}
+        if not movie_info: movie_info = {}
+        return self.getThumbnail(movie_info = movie_info, data=  data, wanted_file_type = 'landscape', i = i)
+
+    def getExtraThumbs(self, movie_info = None, data = None, i = 0):
+        if not data: data = {}
+        if not movie_info: movie_info = {}
+        return self.getThumbnail(movie_info = movie_info, data = data, wanted_file_type = 'extra_thumbs', i = i)
+
+    def getExtraFanart(self, movie_info = None, data = None, i = 0):
+        if not data: data = {}
+        if not movie_info: movie_info = {}
+        return self.getThumbnail(movie_info = movie_info, data = data, wanted_file_type = 'extra_fanart', i = i)

couchpotato/core/media/movie/providers/metadata/mediabrowser.py

@@ -0,0 +1,36 @@
+import os
+
+from couchpotato.core.media.movie.providers.metadata.base import MovieMetaData
+
+
+autoload = 'MediaBrowser'
+
+
+class MediaBrowser(MovieMetaData):
+
+    def getThumbnailName(self, name, root, i):
+        return os.path.join(root, 'folder.jpg')
+
+    def getFanartName(self, name, root, i):
+        return os.path.join(root, 'backdrop.jpg')
+
+
+config = [{
+    'name': 'mediabrowser',
+    'groups': [
+        {
+            'tab': 'renamer',
+            'subtab': 'metadata',
+            'name': 'mediabrowser_metadata',
+            'label': 'MediaBrowser',
+            'description': 'Generate folder.jpg and backdrop.jpg',
+            'options': [
+                {
+                    'name': 'meta_enabled',
+                    'default': False,
+                    'type': 'enabler',
+                },
+            ],
+        },
+    ],
+}]

couchpotato/core/media/movie/providers/metadata/ps3.py

@@ -0,0 +1,33 @@
+import os
+
+from couchpotato.core.media.movie.providers.metadata.base import MovieMetaData
+
+
+autoload = 'SonyPS3'
+
+
+class SonyPS3(MovieMetaData):
+
+    def getThumbnailName(self, name, root, i):
+        return os.path.join(root, 'cover.jpg')
+
+
+config = [{
+    'name': 'sonyps3',
+    'groups': [
+        {
+            'tab': 'renamer',
+            'subtab': 'metadata',
+            'name': 'sonyps3_metadata',
+            'label': 'Sony PS3',
+            'description': 'Generate cover.jpg',
+            'options': [
+                {
+                    'name': 'meta_enabled',
+                    'default': False,
+                    'type': 'enabler',
+                },
+            ],
+        },
+    ],
+}]

couchpotato/core/media/movie/providers/metadata/wmc.py

@@ -8,7 +8,7 @@ autoload = 'WindowsMediaCenter'
 
 class WindowsMediaCenter(MovieMetaData):
 
-    def getThumbnailName(self, name, root):
+    def getThumbnailName(self, name, root, i):
         return os.path.join(root, 'folder.jpg')
 
 

couchpotato/core/media/movie/providers/metadata/xbmc.py

@@ -17,19 +17,43 @@ autoload = 'XBMC'
 
 class XBMC(MovieMetaData):
 
-    def getFanartName(self, name, root):
+    def getFanartName(self, name, root, i):
         return self.createMetaName(self.conf('meta_fanart_name'), name, root)
 
-    def getThumbnailName(self, name, root):
+    def getThumbnailName(self, name, root, i):
         return self.createMetaName(self.conf('meta_thumbnail_name'), name, root)
 
-    def getNfoName(self, name, root):
+    def getNfoName(self, name, root, i):
         return self.createMetaName(self.conf('meta_nfo_name'), name, root)
 
+    def getBannerName(self, name, root, i):
+        return self.createMetaName(self.conf('meta_banner_name'), name, root)
+
+    def getClearArtName(self, name, root, i):
+        return self.createMetaName(self.conf('meta_clear_art_name'), name, root)
+
+    def getLogoName(self, name, root, i):
+        return self.createMetaName(self.conf('meta_logo_name'), name, root)
+
+    def getDiscArtName(self, name, root, i):
+        return self.createMetaName(self.conf('meta_disc_art_name'), name, root)
+
+    def getLandscapeName(self, name, root, i):
+        return self.createMetaName(self.conf('meta_landscape_name'), name, root)
+
+    def getExtraThumbsName(self, name, root, i):
+        return self.createMetaNameMult(self.conf('meta_extra_thumbs_name'), name, root, i)
+
+    def getExtraFanartName(self, name, root, i):
+        return self.createMetaNameMult(self.conf('meta_extra_fanart_name'), name, root, i)
+
     def createMetaName(self, basename, name, root):
         return os.path.join(root, basename.replace('%s', name))
 
-    def getNfo(self, movie_info = None, data = None):
+    def createMetaNameMult(self, basename, name, root, i):
+        return os.path.join(root, basename.replace('%s', name).replace('<i>', str(i + 1)))
+
+    def getNfo(self, movie_info=None, data=None, i=0):
         if not data: data = {}
         if not movie_info: movie_info = {}
 
@@ -129,10 +153,25 @@ class XBMC(MovieMetaData):
         for image_url in movie_info['images']['poster_original']:
             image = SubElement(nfoxml, 'thumb')
             image.text = toUnicode(image_url)
-        fanart = SubElement(nfoxml, 'fanart')
-        for image_url in movie_info['images']['backdrop_original']:
-            image = SubElement(fanart, 'thumb')
-            image.text = toUnicode(image_url)
+
+        image_types = [
+            ('fanart', 'backdrop_original'),
+            ('banner', 'banner'),
+            ('discart', 'disc_art'),
+            ('logo', 'logo'),
+            ('clearart', 'clear_art'),
+            ('landscape', 'landscape'),
+            ('extrathumb', 'extra_thumbs'),
+            ('extrafanart', 'extra_fanart'),
+        ]
+
+        for image_type in image_types:
+            sub, type = image_type
+
+            sub_element = SubElement(nfoxml, sub)
+            for image_url in movie_info['images'][type]:
+                image = SubElement(sub_element, 'thumb')
+                image.text = toUnicode(image_url)
 
         # Add trailer if found
         trailer_found = False
@@ -239,6 +278,92 @@ config = [{
                     'default': '%s.tbn',
                     'advanced': True,
                 },
+                {
+                    'name': 'meta_banner',
+                    'label': 'Banner',
+                    'default': False,
+                    'type': 'bool'
+                },
+                {
+                    'name': 'meta_banner_name',
+                    'label': 'Banner filename',
+                    'default': 'banner.jpg',
+                    'advanced': True,
+                },
+                {
+                    'name': 'meta_clear_art',
+                    'label': 'ClearArt',
+                    'default': False,
+                    'type': 'bool'
+                },
+                {
+                    'name': 'meta_clear_art_name',
+                    'label': 'ClearArt filename',
+                    'default': 'clearart.png',
+                    'advanced': True,
+                },
+                {
+                    'name': 'meta_disc_art',
+                    'label': 'DiscArt',
+                    'default': False,
+                    'type': 'bool'
+                },
+                {
+                    'name': 'meta_disc_art_name',
+                    'label': 'DiscArt filename',
+                    'default': 'disc.png',
+                    'advanced': True,
+                },
+                {
+                    'name': 'meta_landscape',
+                    'label': 'Landscape',
+                    'default': False,
+                    'type': 'bool'
+                },
+                {
+                    'name': 'meta_landscape_name',
+                    'label': 'Landscape filename',
+                    'default': 'landscape.jpg',
+                    'advanced': True,
+                },
+                {
+                    'name': 'meta_logo',
+                    'label': 'ClearLogo',
+                    'default': False,
+                    'type': 'bool'
+                },
+                {
+                    'name': 'meta_logo_name',
+                    'label': 'ClearLogo filename',
+                    'default': 'logo.png',
+                    'advanced': True,
+                },
+                {
+                    'name': 'meta_extra_thumbs',
+                    'label': 'Extrathumbs',
+                    'default': False,
+                    'type': 'bool'
+                },
+                {
+                    'name': 'meta_extra_thumbs_name',
+                    'label': 'Extrathumbs filename',
+                    'description': '&lt;i&gt; is the image number, and must be included to have multiple images',
+                    'default': 'extrathumbs/thumb<i>.jpg',
+                    'advanced': True
+                },
+                {
+                    'name': 'meta_extra_fanart',
+                    'label': 'Extrafanart',
+                    'default': False,
+                    'type': 'bool'
+                },
+                {
+                    'name': 'meta_extra_fanart_name',
+                    'label': 'Extrafanart filename',
+                    'default': 'extrafanart/extrafanart<i>.jpg',
+                    'description': '&lt;i&gt; is the image number, and must be included to have multiple images',
+                    'advanced': True
+                }
             ],
         },
     ],

couchpotato/core/media/movie/providers/nzb/newznab.py

@@ -11,11 +11,16 @@ autoload = 'Newznab'
 
 class Newznab(MovieProvider, Base):
 
-    def buildUrl(self, media, api_key):
+    def buildUrl(self, media, host):
+
         query = tryUrlencode({
             't': 'movie',
             'imdbid': getIdentifier(media).replace('tt', ''),
-            'apikey': api_key,
+            'apikey': host['api_key'],
             'extended': 1
         })
+
+        if len(host.get('custom_tag', '')) > 0:
+            query = '%s&%s' % (query, host.get('custom_tag'))
+
         return query

couchpotato/core/media/movie/providers/nzb/nzbindex.py

@@ -1,30 +0,0 @@
-from couchpotato.core.helpers.encoding import tryUrlencode
-from couchpotato.core.logger import CPLog
-from couchpotato.core.event import fireEvent
-from couchpotato.core.media._base.providers.nzb.nzbindex import Base
-from couchpotato.core.media.movie.providers.base import MovieProvider
-from couchpotato.environment import Env
-
-log = CPLog(__name__)
-
-autoload = 'NzbIndex'
-
-
-class NzbIndex(MovieProvider, Base):
-
-    def buildUrl(self, media, quality):
-        title = fireEvent('library.query', media, include_year = False, single = True)
-        year = media['info']['year']
-
-        query = tryUrlencode({
-            'q': '"%s %s" | "%s (%s)"' % (title, year, title, year),
-            'age': Env.setting('retention', 'nzb'),
-            'sort': 'agedesc',
-            'minsize': quality.get('size_min'),
-            'maxsize': quality.get('size_max'),
-            'rating': 1,
-            'max': 250,
-            'more': 1,
-            'complete': 1,
-        })
-        return query

couchpotato/core/media/movie/providers/torrent/bithdtv.py

@@ -10,10 +10,14 @@ autoload = 'BiTHDTV'
 
 
 class BiTHDTV(MovieProvider, Base):
+    cat_ids = [
+        ([2], ['bd50']),
+    ]
+    cat_backup_id = 7 # Movies
 
-    def buildUrl(self, media):
+    def buildUrl(self, media, quality):
         query = tryUrlencode({
             'search': fireEvent('library.query', media, single = True),
-            'cat': 7  # Movie cat
+            'cat': self.getCatId(quality)[0]
         })
         return query

couchpotato/core/media/movie/providers/torrent/bitsoup.py

@@ -1,6 +1,5 @@
 from couchpotato.core.helpers.encoding import tryUrlencode
 from couchpotato.core.logger import CPLog
-from couchpotato.core.event import fireEvent
 from couchpotato.core.media._base.providers.torrent.bitsoup import Base
 from couchpotato.core.media.movie.providers.base import MovieProvider
 
@@ -18,12 +17,9 @@ class Bitsoup(MovieProvider, Base):
     ]
     cat_backup_id = 0
 
-    def buildUrl(self, media, quality):
+    def buildUrl(self, title, media, quality):
         query = tryUrlencode({
-            'search': '"%s" %s' % (
-                fireEvent('library.query', media, include_year = False, single = True),
-                media['info']['year']
-            ),
+            'search': '"%s" %s' % (title, media['info']['year']),
             'cat': self.getCatId(quality)[0],
         })
         return query

couchpotato/core/media/movie/providers/torrent/iptorrents.py

@@ -13,11 +13,11 @@ class IPTorrents(MovieProvider, Base):
         ([87], ['3d']),
         ([48], ['720p', '1080p', 'bd50']),
         ([72], ['cam', 'ts', 'tc', 'r5', 'scr']),
-        ([7], ['dvdrip', 'brrip']),
+        ([7,48], ['dvdrip', 'brrip']),
         ([6], ['dvdr']),
     ]
 
     def buildUrl(self, title, media, quality):
-        query = '%s %s' % (title.replace(':', ''), media['info']['year'])
+        query = '"%s" %s' % (title.replace(':', ''), media['info']['year'])
 
         return self._buildUrl(query, quality)

couchpotato/core/media/movie/providers/torrent/passthepopcorn.py

@@ -13,7 +13,7 @@ class PassThePopcorn(MovieProvider, Base):
         'bd50': {'media': 'Blu-ray', 'format': 'BD50'},
         '1080p': {'resolution': '1080p'},
         '720p': {'resolution': '720p'},
-        'brrip': {'media': 'Blu-ray'},
+        'brrip': {'resolution': 'anyhd'},
         'dvdr': {'resolution': 'anysd'},
         'dvdrip': {'media': 'DVD'},
         'scr': {'media': 'DVD-Screener'},
@@ -27,7 +27,7 @@ class PassThePopcorn(MovieProvider, Base):
         'bd50': {'Codec': ['BD50']},
         '1080p': {'Resolution': ['1080p']},
         '720p': {'Resolution': ['720p']},
-        'brrip': {'Source': ['Blu-ray'], 'Quality': ['High Definition'], 'Container': ['!ISO']},
+        'brrip': {'Quality': ['High Definition'], 'Container': ['!ISO']},
         'dvdr': {'Codec': ['DVD5', 'DVD9']},
         'dvdrip': {'Source': ['DVD'], 'Codec': ['!DVD5', '!DVD9']},
         'scr': {'Source': ['DVD-Screener']},

couchpotato/core/media/movie/providers/torrent/publichd.py

@@ -1,14 +0,0 @@
-from couchpotato.core.logger import CPLog
-from couchpotato.core.event import fireEvent
-from couchpotato.core.media._base.providers.torrent.publichd import Base
-from couchpotato.core.media.movie.providers.base import MovieProvider
-
-log = CPLog(__name__)
-
-autoload = 'PublicHD'
-
-
-class PublicHD(MovieProvider, Base):
-
-    def buildUrl(self, media):
-        return fireEvent('library.query', media, single = True)

couchpotato/core/media/movie/providers/torrent/sceneaccess.py

@@ -1,5 +1,4 @@
 from couchpotato.core.helpers.encoding import tryUrlencode
-from couchpotato.core.event import fireEvent
 from couchpotato.core.logger import CPLog
 from couchpotato.core.media._base.providers.torrent.sceneaccess import Base
 from couchpotato.core.media.movie.providers.base import MovieProvider
@@ -17,13 +16,13 @@ class SceneAccess(MovieProvider, Base):
         ([8], ['dvdr']),
     ]
 
-    def buildUrl(self, media, quality):
+    def buildUrl(self, title, media, quality):
         cat_id = self.getCatId(quality)[0]
         url = self.urls['search'] % (cat_id, cat_id)
 
         arguments = tryUrlencode({
-            'search': fireEvent('library.query', media, single = True),
-            'method': 3,
+            'search': '%s %s' % (title, media['info']['year']),
+            'method': 2,
         })
         query = "%s&%s" % (url, arguments)
 

couchpotato/core/media/movie/providers/torrent/thepiratebay.py

@@ -13,7 +13,7 @@ class ThePirateBay(MovieProvider, Base):
 
     cat_ids = [
         ([209], ['3d']),
-        ([207], ['720p', '1080p']),
+        ([207], ['720p', '1080p', 'bd50']),
         ([201], ['cam', 'ts', 'dvdrip', 'tc', 'r5', 'scr']),
         ([201, 207], ['brrip']),
         ([202], ['dvdr'])

couchpotato/core/media/movie/providers/torrent/torrentday.py

@@ -1,5 +1,4 @@
 from couchpotato.core.logger import CPLog
-from couchpotato.core.event import fireEvent
 from couchpotato.core.media._base.providers.torrent.torrentday import Base
 from couchpotato.core.media.movie.providers.base import MovieProvider
 
@@ -16,6 +15,3 @@ class TorrentDay(MovieProvider, Base):
         ([3], ['dvdr']),
         ([5], ['bd50']),
     ]
-
-    def buildUrl(self, media):
-        return fireEvent('library.query', media, single = True)

couchpotato/core/media/movie/providers/torrent/torrentleech.py

@@ -1,4 +1,3 @@
-from couchpotato import fireEvent
 from couchpotato.core.helpers.encoding import tryUrlencode
 from couchpotato.core.logger import CPLog
 from couchpotato.core.media._base.providers.torrent.torrentleech import Base
@@ -12,7 +11,7 @@ autoload = 'TorrentLeech'
 class TorrentLeech(MovieProvider, Base):
 
     cat_ids = [
-        ([13], ['720p', '1080p']),
+        ([13], ['720p', '1080p', 'bd50']),
         ([8], ['cam']),
         ([9], ['ts', 'tc']),
         ([10], ['r5', 'scr']),
@@ -21,8 +20,8 @@ class TorrentLeech(MovieProvider, Base):
         ([12], ['dvdr']),
     ]
 
-    def buildUrl(self, media, quality):
+    def buildUrl(self, title, media, quality):
         return (
-            tryUrlencode(fireEvent('library.query', media, single = True)),
+            tryUrlencode(title.replace(':', '')),
             self.getCatId(quality)[0]
         )

couchpotato/core/media/movie/providers/torrent/torrentshack.py

@@ -31,6 +31,6 @@ class TorrentShack(MovieProvider, Base):
 
     def buildUrl(self, media, quality):
         query = (tryUrlencode(fireEvent('library.query', media, single = True)),
-                 self.getCatId(quality)[0],
-                 self.getSceneOnly())
+                 self.getSceneOnly(),
+                 self.getCatId(quality)[0])
         return query

couchpotato/core/media/movie/providers/trailer/base.py

@@ -11,3 +11,6 @@ class TrailerProvider(Provider):
 
     def __init__(self):
         addEvent('trailer.search', self.search)
+
+    def search(self, *args, **kwargs):
+        pass