Merge branch 'develop'

fix #4674

CouchPotato.py

@@ -61,7 +61,7 @@ class Loader(object):
         self.log = CPLog(__name__)
 
         formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s', '%H:%M:%S')
-        hdlr = handlers.RotatingFileHandler(os.path.join(self.log_dir, 'error.log'), 'a', 500000, 10, encoding = 'utf-8')
+        hdlr = handlers.RotatingFileHandler(os.path.join(self.log_dir, 'error.log'), 'a', 500000, 10)
         hdlr.setLevel(logging.CRITICAL)
         hdlr.setFormatter(formatter)
         self.log.logger.addHandler(hdlr)

couchpotato/__init__.py

@@ -40,6 +40,8 @@ class WebHandler(BaseHandler):
             return
 
         try:
+            if route == 'robots.txt':
+                self.set_header('Content-Type', 'text/plain')
             self.write(views[route]())
         except:
             log.error("Failed doing web request '%s': %s", (route, traceback.format_exc()))
@@ -60,6 +62,13 @@ def index():
 addView('', index)
 
 
+# Web view
+def robots():
+    return 'User-agent: * \n' \
+           'Disallow: /'
+addView('robots.txt', robots)
+
+
 # API docs
 def apiDocs():
     routes = list(api.keys())

couchpotato/api.py

@@ -3,10 +3,11 @@ from threading import Thread
 import json
 import threading
 import traceback
-from six.moves import urllib
+import urllib
 
 from couchpotato.core.helpers.request import getParams
 from couchpotato.core.logger import CPLog
+from tornado.ioloop import IOLoop
 from tornado.web import RequestHandler, asynchronous
 
 
@@ -50,24 +51,22 @@ class NonBlockHandler(RequestHandler):
         start, stop = api_nonblock[route]
         self.stopper = stop
 
-        start(self.onNewMessage, last_id = self.get_argument('last_id', None))
+        start(self.sendData, last_id = self.get_argument('last_id', None))
 
-    def onNewMessage(self, response):
+    def sendData(self, response):
-        if self.request.connection.stream.closed():
+        if not self.request.connection.stream.closed():
-            self.on_connection_close()
+            try:
-            return
+                self.finish(response)
+            except:
+                log.debug('Failed doing nonblock request, probably already closed: %s', (traceback.format_exc()))
+                try: self.finish({'success': False, 'error': 'Failed returning results'})
+                except: pass
 
-        try:
+        self.removeStopper()
-            self.finish(response)
-        except:
-            log.debug('Failed doing nonblock request, probably already closed: %s', (traceback.format_exc()))
-            try: self.finish({'success': False, 'error': 'Failed returning results'})
-            except: pass
-
-    def on_connection_close(self):
 
+    def removeStopper(self):
         if self.stopper:
-            self.stopper(self.onNewMessage)
+            self.stopper(self.sendData)
 
         self.stopper = None
 
@@ -83,10 +82,11 @@ def addNonBlockApiView(route, func_tuple, docs = None, **kwargs):
 
 # Blocking API handler
 class ApiHandler(RequestHandler):
+    route = None
 
     @asynchronous
     def get(self, route, *args, **kwargs):
-        route = route.strip('/')
+        self.route = route = route.strip('/')
         if not api.get(route):
             self.write('API call doesn\'t seem to exist')
             self.finish()
@@ -102,7 +102,7 @@ class ApiHandler(RequestHandler):
 
             kwargs = {}
             for x in self.request.arguments:
-                kwargs[x] = urllib.parse.unquote(self.get_argument(x))
+                kwargs[x] = urllib.unquote(self.get_argument(x))
 
             # Split array arguments
             kwargs = getParams(kwargs)
@@ -123,11 +123,15 @@ class ApiHandler(RequestHandler):
             except:
                 log.error('Failed write error "%s": %s', (route, traceback.format_exc()))
 
-            api_locks[route].release()
+            self.unlock()
 
     post = get
 
     def taskFinished(self, result, route):
+        IOLoop.current().add_callback(self.sendData, result, route)
+        self.unlock()
+
+    def sendData(self, result, route):
 
         if not self.request.connection.stream.closed():
             try:
@@ -135,14 +139,12 @@ class ApiHandler(RequestHandler):
                 jsonp_callback = self.get_argument('callback_func', default = None)
 
                 if jsonp_callback:
-                    self.write(str(jsonp_callback) + '(' + json.dumps(result) + ')')
+                    self.set_header('Content-Type', 'text/javascript')
-                    self.set_header("Content-Type", "text/javascript")
+                    self.finish(str(jsonp_callback) + '(' + json.dumps(result) + ')')
-                    self.finish()
                 elif isinstance(result, tuple) and result[0] == 'redirect':
                     self.redirect(result[1])
                 else:
-                    self.write(result)
+                    self.finish(result)
-                    self.finish()
             except UnicodeDecodeError:
                 log.error('Failed proper encode: %s', traceback.format_exc())
             except:
@@ -150,7 +152,9 @@ class ApiHandler(RequestHandler):
                 try: self.finish({'success': False, 'error': 'Failed returning results'})
                 except: pass
 
-        api_locks[route].release()
+    def unlock(self):
+        try: api_locks[self.route].release()
+        except: pass
 
 
 def addApiView(route, func, static = False, docs = None, **kwargs):

couchpotato/core/_base/_core.py

@@ -181,13 +181,13 @@ class Core(Plugin):
         return '%sapi/%s' % (self.createBaseUrl(), Env.setting('api_key'))
 
     def version(self):
-        ver = fireEvent('updater.info', single = True)
+        ver = fireEvent('updater.info', single = True) or {'version': {}}
 
         if os.name == 'nt': platf = 'windows'
         elif 'Darwin' in platform.platform(): platf = 'osx'
         else: platf = 'linux'
 
-        return '%s - %s-%s - v2' % (platf, ver.get('version')['type'], ver.get('version')['hash'])
+        return '%s - %s-%s - v2' % (platf, ver.get('version').get('type') or 'unknown', ver.get('version').get('hash') or 'unknown')
 
     def versionView(self, **kwargs):
         return {
@@ -290,7 +290,7 @@ config = [{
                 },
                 {
                     'name': 'permission_file',
-                    'default': '0755',
+                    'default': '0644',
                     'label': 'File CHMOD',
                     'description': 'See Folder CHMOD description, but for files',
                 },

couchpotato/core/_base/updater/main.py

@@ -205,19 +205,28 @@ class GitUpdater(BaseUpdater):
     def getVersion(self):
 
         if not self.version:
+
+            hash = None
+            date = None
+            branch = self.branch
+
             try:
                 output = self.repo.getHead()  # Yes, please
                 log.debug('Git version output: %s', output.hash)
-                self.version = {
+
-                    'repr': 'git:(%s:%s % s) %s (%s)' % (self.repo_user, self.repo_name, self.repo.getCurrentBranch().name or self.branch, output.hash[:8], datetime.fromtimestamp(output.getDate())),
+                hash = output.hash[:8]
-                    'hash': output.hash[:8],
+                date = output.getDate()
-                    'date': output.getDate(),
+                branch = self.repo.getCurrentBranch().name
-                    'type': 'git',
-                    'branch': self.repo.getCurrentBranch().name
-                }
             except Exception as e:
                 log.error('Failed using GIT updater, running from source, you need to have GIT installed. %s', e)
-                return 'No GIT'
+
+            self.version = {
+                'repr': 'git:(%s:%s % s) %s (%s)' % (self.repo_user, self.repo_name, branch, hash or 'unknown_hash', datetime.fromtimestamp(date) if date else 'unknown_date'),
+                'hash': hash,
+                'date': date,
+                'type': 'git',
+                'branch': branch
+            }
 
         return self.version
 

couchpotato/core/database.py

@@ -3,12 +3,12 @@ import os
 import time
 import traceback
 from sqlite3 import OperationalError
-from CodernityDB3.index import Index
 
+from CodernityDB.database import RecordNotFound
+from CodernityDB.index import IndexException, IndexNotFoundException, IndexConflict
 from couchpotato import CPLog
 from couchpotato.api import addApiView
 from couchpotato.core.event import addEvent, fireEvent, fireEventAsync
-from couchpotato.core.helpers.database import IndexException, IndexNotFoundException, IndexConflict, RecordNotFound
 from couchpotato.core.helpers.encoding import toUnicode, sp
 from couchpotato.core.helpers.variable import getImdb, tryInt, randomString
 
@@ -621,6 +621,8 @@ class Database(object):
 
             except OperationalError:
                 log.error('Migrating from faulty database, probably a (too) old version: %s', traceback.format_exc())
+                
+                rename_old = True
             except:
                 log.error('Migration failed: %s', traceback.format_exc())
 

couchpotato/core/downloaders/blackhole.py

@@ -20,14 +20,31 @@ class Blackhole(DownloaderBase):
     status_support = False
 
     def download(self, data = None, media = None, filedata = None):
+        """ Send a torrent/nzb file to the downloader
+
+        :param data: dict returned from provider
+            Contains the release information
+        :param media: media dict with information
+            Used for creating the filename when possible
+        :param filedata: downloaded torrent/nzb filedata
+            The file gets downloaded in the searcher and send to this function
+            This is done to have failed checking before using the downloader, so the downloader
+            doesn't need to worry about that
+        :return: boolean
+            One faile returns false, but the downloaded should log his own errors
+        """
+
         if not media: media = {}
         if not data: data = {}
 
         directory = self.conf('directory')
+
+        # The folder needs to exist
         if not directory or not os.path.isdir(directory):
             log.error('No directory set for blackhole %s download.', data.get('protocol'))
         else:
             try:
+                # Filedata can be empty, which probably means it a magnet link
                 if not filedata or len(filedata) < 50:
                     try:
                         if data.get('protocol') == 'torrent_magnet':
@@ -36,13 +53,16 @@ class Blackhole(DownloaderBase):
                     except:
                         log.error('Failed download torrent via magnet url: %s', traceback.format_exc())
 
+                    # If it's still empty, don't know what to do!
                     if not filedata or len(filedata) < 50:
                         log.error('No nzb/torrent available: %s', data.get('url'))
                         return False
 
+                # Create filename with imdb id and other nice stuff
                 file_name = self.createFileName(data, filedata, media)
                 full_path = os.path.join(directory, file_name)
 
+                # People want thinks nice and tidy, create a subdir
                 if self.conf('create_subdir'):
                     try:
                         new_path = os.path.splitext(full_path)[0]
@@ -53,6 +73,8 @@ class Blackhole(DownloaderBase):
                         log.error('Couldnt create sub dir, reverting to old one: %s', full_path)
 
                 try:
+
+                    # Make sure the file doesn't exist yet, no need in overwriting it
                     if not os.path.isfile(full_path):
                         log.info('Downloading %s to %s.', (data.get('protocol'), full_path))
                         with open(full_path, 'wb') as f:
@@ -74,6 +96,10 @@ class Blackhole(DownloaderBase):
         return False
 
     def test(self):
+        """ Test and see if the directory is writable
+        :return: boolean
+        """
+
         directory = self.conf('directory')
         if directory and os.path.isdir(directory):
 
@@ -88,6 +114,10 @@ class Blackhole(DownloaderBase):
         return False
 
     def getEnabledProtocol(self):
+        """ What protocols is this downloaded used for
+        :return: list with protocols
+        """
+
         if self.conf('use_for') == 'both':
             return super(Blackhole, self).getEnabledProtocol()
         elif self.conf('use_for') == 'torrent':
@@ -96,6 +126,12 @@ class Blackhole(DownloaderBase):
             return ['nzb']
 
     def isEnabled(self, manual = False, data = None):
+        """ Check if protocol is used (and enabled)
+        :param manual: The user has clicked to download a link through the webUI
+        :param data: dict returned from provider
+            Contains the release information
+        :return: boolean
+        """
         if not data: data = {}
         for_protocol = ['both']
         if data and 'torrent' in data.get('protocol'):

couchpotato/core/downloaders/deluge.py

@@ -25,8 +25,18 @@ class Deluge(DownloaderBase):
     drpc = None
 
     def connect(self, reconnect = False):
+        """ Connect to the delugeRPC, re-use connection when already available
+        :param reconnect: force reconnect
+        :return: DelugeRPC instance
+        """
+
         # Load host from config and split out port.
         host = cleanHost(self.conf('host'), protocol = False).split(':')
+
+        # Force host assignment
+        if len(host) == 1:
+            host.append(80)
+
         if not isInt(host[1]):
             log.error('Config properties are not filled in correctly, port is missing.')
             return False
@@ -37,6 +47,20 @@ class Deluge(DownloaderBase):
         return self.drpc
 
     def download(self, data = None, media = None, filedata = None):
+        """ Send a torrent/nzb file to the downloader
+
+        :param data: dict returned from provider
+            Contains the release information
+        :param media: media dict with information
+            Used for creating the filename when possible
+        :param filedata: downloaded torrent/nzb filedata
+            The file gets downloaded in the searcher and send to this function
+            This is done to have failed checking before using the downloader, so the downloader
+            doesn't need to worry about that
+        :return: boolean
+            One faile returns false, but the downloaded should log his own errors
+        """
+
         if not media: media = {}
         if not data: data = {}
 
@@ -91,11 +115,21 @@ class Deluge(DownloaderBase):
         return self.downloadReturnId(remote_torrent)
 
     def test(self):
+        """ Check if connection works
+        :return: bool
+        """
         if self.connect(True) and self.drpc.test():
             return True
         return False
 
     def getAllDownloadStatus(self, ids):
+        """ Get status of all active downloads
+
+        :param ids: list of (mixed) downloader ids
+            Used to match the releases for this downloader as there could be
+            other downloaders active that it should ignore
+        :return: list of releases
+        """
 
         log.debug('Checking Deluge download status.')
 

couchpotato/core/downloaders/hadouken.py

@@ -0,0 +1,427 @@
+from base64 import b16encode, b32decode, b64encode
+from distutils.version import LooseVersion
+from hashlib import sha1
+import httplib
+import json
+import os
+import re
+import urllib2
+
+from couchpotato.core._base.downloader.main import DownloaderBase, ReleaseDownloadList
+from couchpotato.core.helpers.encoding import isInt, sp
+from couchpotato.core.helpers.variable import cleanHost
+from couchpotato.core.logger import CPLog
+from bencode import bencode as benc, bdecode
+
+
+log = CPLog(__name__)
+
+autoload = 'Hadouken'
+
+
+class Hadouken(DownloaderBase):
+    protocol = ['torrent', 'torrent_magnet']
+    hadouken_api = None
+
+    def connect(self):
+        # Load host from config and split out port.
+        host = cleanHost(self.conf('host'), protocol = False).split(':')
+
+        if not isInt(host[1]):
+            log.error('Config properties are not filled in correctly, port is missing.')
+            return False
+
+        if not self.conf('api_key'):
+            log.error('Config properties are not filled in correctly, API key is missing.')
+            return False
+
+        self.hadouken_api = HadoukenAPI(host[0], port = host[1], api_key = self.conf('api_key'))
+
+        return True
+
+    def download(self, data = None, media = None, filedata = None):
+        """ Send a torrent/nzb file to the downloader
+
+        :param data: dict returned from provider
+            Contains the release information
+        :param media: media dict with information
+            Used for creating the filename when possible
+        :param filedata: downloaded torrent/nzb filedata
+            The file gets downloaded in the searcher and send to this function
+            This is done to have failed checking before using the downloader, so the downloader
+            doesn't need to worry about that
+        :return: boolean
+            One faile returns false, but the downloaded should log his own errors
+        """
+
+        if not media: media = {}
+        if not data: data = {}
+
+        log.debug("Sending '%s' (%s) to Hadouken.", (data.get('name'), data.get('protocol')))
+
+        if not self.connect():
+            return False
+
+        torrent_params = {}
+
+        if self.conf('label'):
+            torrent_params['label'] = self.conf('label')
+
+        torrent_filename = self.createFileName(data, filedata, media)
+
+        if data.get('protocol') == 'torrent_magnet':
+            torrent_hash = re.findall('urn:btih:([\w]{32,40})', data.get('url'))[0].upper()
+            torrent_params['trackers'] = self.torrent_trackers
+            torrent_params['name'] = torrent_filename
+        else:
+            info = bdecode(filedata)['info']
+            torrent_hash = sha1(benc(info)).hexdigest().upper()
+
+        # Convert base 32 to hex
+        if len(torrent_hash) == 32:
+            torrent_hash = b16encode(b32decode(torrent_hash))
+
+        # Send request to Hadouken
+        if data.get('protocol') == 'torrent_magnet':
+            self.hadouken_api.add_magnet_link(data.get('url'), torrent_params)
+        else:
+            self.hadouken_api.add_file(filedata, torrent_params)
+
+        return self.downloadReturnId(torrent_hash)
+
+    def test(self):
+        """ Tests the given host:port and API key """
+
+        if not self.connect():
+            return False
+
+        version = self.hadouken_api.get_version()
+
+        if not version:
+            log.error('Could not get Hadouken version.')
+            return False
+
+        # The minimum required version of Hadouken is 4.5.6.
+        if LooseVersion(version) >= LooseVersion('4.5.6'):
+            return True
+
+        log.error('Hadouken v4.5.6 (or newer) required. Found v%s', version)
+        return False
+
+    def getAllDownloadStatus(self, ids):
+        """ Get status of all active downloads
+
+        :param ids: list of (mixed) downloader ids
+            Used to match the releases for this downloader as there could be
+            other downloaders active that it should ignore
+        :return: list of releases
+        """
+
+        log.debug('Checking Hadouken download status.')
+
+        if not self.connect():
+            return []
+
+        release_downloads = ReleaseDownloadList(self)
+        queue = self.hadouken_api.get_by_hash_list(ids)
+
+        if not queue:
+            return []
+
+        for torrent in queue:
+            if torrent is None:
+                continue
+
+            torrent_filelist = self.hadouken_api.get_files_by_hash(torrent['InfoHash'])
+            torrent_files = []
+
+            save_path = torrent['SavePath']
+
+            # The 'Path' key for each file_item contains
+            # the full path to the single file relative to the
+            # torrents save path.
+
+            # For a single file torrent the result would be,
+            # - Save path: "C:\Downloads"
+            # - file_item['Path'] = "file1.iso"
+            # Resulting path: "C:\Downloads\file1.iso"
+
+            # For a multi file torrent the result would be,
+            # - Save path: "C:\Downloads"
+            # - file_item['Path'] = "dirname/file1.iso"
+            # Resulting path: "C:\Downloads\dirname/file1.iso"
+
+            for file_item in torrent_filelist:
+                torrent_files.append(sp(os.path.join(save_path, file_item['Path'])))
+
+            release_downloads.append({
+                'id': torrent['InfoHash'].upper(),
+                'name': torrent['Name'],
+                'status': self.get_torrent_status(torrent),
+                'seed_ratio': self.get_seed_ratio(torrent),
+                'original_status': torrent['State'],
+                'timeleft': -1,
+                'folder': sp(save_path if len(torrent_files == 1) else os.path.join(save_path, torrent['Name'])),
+                'files': torrent_files
+            })
+
+        return release_downloads
+
+    def get_seed_ratio(self, torrent):
+        """ Returns the seed ratio for a given torrent.
+
+        Keyword arguments:
+        torrent -- The torrent to calculate seed ratio for.
+        """
+
+        up = torrent['TotalUploadedBytes']
+        down = torrent['TotalDownloadedBytes']
+
+        if up > 0 and down > 0:
+            return up / down
+
+        return 0
+
+    def get_torrent_status(self, torrent):
+        """ Returns the CouchPotato status for a given torrent.
+
+        Keyword arguments:
+        torrent -- The torrent to translate status for.
+        """
+
+        if torrent['IsSeeding'] and torrent['IsFinished'] and torrent['Paused']:
+            return 'completed'
+
+        if torrent['IsSeeding']:
+            return 'seeding'
+
+        return 'busy'
+
+    def pause(self, release_download, pause = True):
+        """ Pauses or resumes the torrent specified by the ID field
+        in release_download.
+
+        Keyword arguments:
+        release_download -- The CouchPotato release_download to pause/resume.
+        pause -- Boolean indicating whether to pause or resume.
+        """
+
+        if not self.connect():
+            return False
+
+        return self.hadouken_api.pause(release_download['id'], pause)
+
+    def removeFailed(self, release_download):
+        """ Removes a failed torrent and also remove the data associated with it.
+
+        Keyword arguments:
+        release_download -- The CouchPotato release_download to remove.
+        """
+
+        log.info('%s failed downloading, deleting...', release_download['name'])
+
+        if not self.connect():
+            return False
+
+        return self.hadouken_api.remove(release_download['id'], remove_data = True)
+
+    def processComplete(self, release_download, delete_files = False):
+        """ Removes the completed torrent from Hadouken and optionally removes the data
+        associated with it.
+
+        Keyword arguments:
+        release_download -- The CouchPotato release_download to remove.
+        delete_files: Boolean indicating whether to remove the associated data.
+        """
+
+        log.debug('Requesting Hadouken to remove the torrent %s%s.',
+                  (release_download['name'], ' and cleanup the downloaded files' if delete_files else ''))
+
+        if not self.connect():
+            return False
+
+        return self.hadouken_api.remove(release_download['id'], remove_data = delete_files)
+
+
+class HadoukenAPI(object):
+    def __init__(self, host = 'localhost', port = 7890, api_key = None):
+        self.url = 'http://' + str(host) + ':' + str(port)
+        self.api_key = api_key
+        self.requestId = 0;
+
+        self.opener = urllib2.build_opener()
+        self.opener.addheaders = [('User-agent', 'couchpotato-hadouken-client/1.0'), ('Accept', 'application/json')]
+
+        if not api_key:
+            log.error('API key missing.')
+
+    def add_file(self, filedata, torrent_params):
+        """ Add a file to Hadouken with the specified parameters.
+
+        Keyword arguments:
+        filedata -- The binary torrent data.
+        torrent_params -- Additional parameters for the file.
+        """
+        data = {
+            'method': 'torrents.addFile',
+            'params': [b64encode(filedata), torrent_params]
+        }
+
+        return self._request(data)
+
+    def add_magnet_link(self, magnetLink, torrent_params):
+        """ Add a magnet link to Hadouken with the specified parameters.
+
+        Keyword arguments:
+        magnetLink -- The magnet link to send.
+        torrent_params -- Additional parameters for the magnet link.
+        """
+        data = {
+            'method': 'torrents.addUrl',
+            'params': [magnetLink, torrent_params]
+        }
+
+        return self._request(data)
+
+    def get_by_hash_list(self, infoHashList):
+        """ Gets a list of torrents filtered by the given info hash list.
+
+        Keyword arguments:
+        infoHashList -- A list of info hashes.
+        """
+        data = {
+            'method': 'torrents.getByInfoHashList',
+            'params': [infoHashList]
+        }
+
+        return self._request(data)
+
+    def get_files_by_hash(self, infoHash):
+        """ Gets a list of files for the torrent identified by the
+        given info hash.
+
+        Keyword arguments:
+        infoHash -- The info hash of the torrent to return files for.
+        """
+        data = {
+            'method': 'torrents.getFiles',
+            'params': [infoHash]
+        }
+
+        return self._request(data)
+
+    def get_version(self):
+        """ Gets the version, commitish and build date of Hadouken. """
+        data = {
+            'method': 'core.getVersion',
+            'params': None
+        }
+
+        result = self._request(data)
+
+        if not result:
+            return False
+
+        return result['Version']
+
+    def pause(self, infoHash, pause):
+        """ Pauses/unpauses the torrent identified by the given info hash.
+
+        Keyword arguments:
+        infoHash -- The info hash of the torrent to operate on.
+        pause -- If true, pauses the torrent. Otherwise resumes.
+        """
+        data = {
+            'method': 'torrents.pause',
+            'params': [infoHash]
+        }
+
+        if not pause:
+            data['method'] = 'torrents.resume'
+
+        return self._request(data)
+
+    def remove(self, infoHash, remove_data = False):
+        """ Removes the torrent identified by the given info hash and
+        optionally removes the data as well.
+
+        Keyword arguments:
+        infoHash -- The info hash of the torrent to remove.
+        remove_data -- If true, removes the data associated with the torrent.
+        """
+        data = {
+            'method': 'torrents.remove',
+            'params': [infoHash, remove_data]
+        }
+
+        return self._request(data)
+
+
+    def _request(self, data):
+        self.requestId += 1
+
+        data['jsonrpc'] = '2.0'
+        data['id'] = self.requestId
+
+        request = urllib2.Request(self.url + '/jsonrpc', data = json.dumps(data))
+        request.add_header('Authorization', 'Token ' + self.api_key)
+        request.add_header('Content-Type', 'application/json')
+
+        try:
+            f = self.opener.open(request)
+            response = f.read()
+            f.close()
+
+            obj = json.loads(response)
+
+            if not 'error' in obj.keys():
+                return obj['result']
+
+            log.error('JSONRPC error, %s: %s', obj['error']['code'], obj['error']['message'])
+        except httplib.InvalidURL as err:
+            log.error('Invalid Hadouken host, check your config %s', err)
+        except urllib2.HTTPError as err:
+            if err.code == 401:
+                log.error('Invalid Hadouken API key, check your config')
+            else:
+                log.error('Hadouken HTTPError: %s', err)
+        except urllib2.URLError as err:
+            log.error('Unable to connect to Hadouken %s', err)
+
+        return False
+
+
+config = [{
+    'name': 'hadouken',
+    'groups': [
+        {
+            'tab': 'downloaders',
+            'list': 'download_providers',
+            'name': 'hadouken',
+            'label': 'Hadouken',
+            'description': 'Use <a href="http://www.hdkn.net">Hadouken</a> (>= v4.5.6) to download torrents.',
+            'wizard': True,
+            'options': [
+                {
+                    'name': 'enabled',
+                    'default': 0,
+                    'type': 'enabler',
+                    'radio_group': 'torrent'
+                },
+                {
+                    'name': 'host',
+                    'default': 'localhost:7890'
+                },
+                {
+                    'name': 'api_key',
+                    'label': 'API key',
+                    'type': 'password'
+                },
+                {
+                    'name': 'label',
+                    'description': 'Label to add torrent as.'
+                }
+            ]
+        }
+    ]
+}]

couchpotato/core/downloaders/nzbget.py

@@ -23,6 +23,20 @@ class NZBGet(DownloaderBase):
     rpc = 'xmlrpc'
 
     def download(self, data = None, media = None, filedata = None):
+        """ Send a torrent/nzb file to the downloader
+
+        :param data: dict returned from provider
+            Contains the release information
+        :param media: media dict with information
+            Used for creating the filename when possible
+        :param filedata: downloaded torrent/nzb filedata
+            The file gets downloaded in the searcher and send to this function
+            This is done to have failed checking before using the downloader, so the downloader
+            doesn't need to worry about that
+        :return: boolean
+            One faile returns false, but the downloaded should log his own errors
+        """
+
         if not media: media = {}
         if not data: data = {}
 
@@ -71,6 +85,10 @@ class NZBGet(DownloaderBase):
             return False
 
     def test(self):
+        """ Check if connection works
+        :return: bool
+        """
+
         rpc = self.getRPC()
 
         try:
@@ -91,6 +109,13 @@ class NZBGet(DownloaderBase):
         return True
 
     def getAllDownloadStatus(self, ids):
+        """ Get status of all active downloads
+
+        :param ids: list of (mixed) downloader ids
+            Used to match the releases for this downloader as there could be
+            other downloaders active that it should ignore
+        :return: list of releases
+        """
 
         log.debug('Checking NZBGet download status.')
 
@@ -163,12 +188,12 @@ class NZBGet(DownloaderBase):
                 nzb_id = nzb['NZBID']
 
             if nzb_id in ids:
-                log.debug('Found %s in NZBGet history. ParStatus: %s, ScriptStatus: %s, Log: %s', (nzb['NZBFilename'] , nzb['ParStatus'], nzb['ScriptStatus'] , nzb['Log']))
+                log.debug('Found %s in NZBGet history. TotalStatus: %s, ParStatus: %s, ScriptStatus: %s, Log: %s', (nzb['NZBFilename'] , nzb['Status'], nzb['ParStatus'], nzb['ScriptStatus'] , nzb['Log']))
                 release_downloads.append({
                     'id': nzb_id,
                     'name': nzb['NZBFilename'],
-                    'status': 'completed' if nzb['ParStatus'] in ['SUCCESS', 'NONE'] and nzb['ScriptStatus'] in ['SUCCESS', 'NONE'] else 'failed',
+                    'status': 'completed' if 'SUCCESS' in nzb['Status'] else 'failed',
-                    'original_status': nzb['ParStatus'] + ', ' + nzb['ScriptStatus'],
+                    'original_status': nzb['Status'],
                     'timeleft': str(timedelta(seconds = 0)),
                     'folder': sp(nzb['DestDir'])
                 })

couchpotato/core/downloaders/nzbvortex.py

@@ -24,6 +24,20 @@ class NZBVortex(DownloaderBase):
     session_id = None
 
     def download(self, data = None, media = None, filedata = None):
+        """ Send a torrent/nzb file to the downloader
+
+        :param data: dict returned from provider
+            Contains the release information
+        :param media: media dict with information
+            Used for creating the filename when possible
+        :param filedata: downloaded torrent/nzb filedata
+            The file gets downloaded in the searcher and send to this function
+            This is done to have failed checking before using the downloader, so the downloader
+            doesn't need to worry about that
+        :return: boolean
+            One faile returns false, but the downloaded should log his own errors
+        """
+
         if not media: media = {}
         if not data: data = {}
 
@@ -45,6 +59,10 @@ class NZBVortex(DownloaderBase):
             return False
 
     def test(self):
+        """ Check if connection works
+        :return: bool
+        """
+
         try:
             login_result = self.login()
         except:
@@ -53,6 +71,13 @@ class NZBVortex(DownloaderBase):
         return login_result
 
     def getAllDownloadStatus(self, ids):
+        """ Get status of all active downloads
+
+        :param ids: list of (mixed) downloader ids
+            Used to match the releases for this downloader as there could be
+            other downloaders active that it should ignore
+        :return: list of releases
+        """
 
         raw_statuses = self.call('nzb')
 

couchpotato/core/downloaders/pneumatic.py

@@ -19,6 +19,20 @@ class Pneumatic(DownloaderBase):
     status_support = False
 
     def download(self, data = None, media = None, filedata = None):
+        """ Send a torrent/nzb file to the downloader
+
+        :param data: dict returned from provider
+            Contains the release information
+        :param media: media dict with information
+            Used for creating the filename when possible
+        :param filedata: downloaded torrent/nzb filedata
+            The file gets downloaded in the searcher and send to this function
+            This is done to have failed checking before using the downloader, so the downloader
+            doesn't need to worry about that
+        :return: boolean
+            One faile returns false, but the downloaded should log his own errors
+        """
+
         if not media: media = {}
         if not data: data = {}
 
@@ -63,6 +77,10 @@ class Pneumatic(DownloaderBase):
         return False
 
     def test(self):
+        """ Check if connection works
+        :return: bool
+        """
+
         directory = self.conf('directory')
         if directory and os.path.isdir(directory):
 

couchpotato/core/downloaders/putio/__init__.py

@@ -0,0 +1,68 @@
+from .main import PutIO
+
+
+def autoload():
+    return PutIO()
+
+
+config = [{
+    'name': 'putio',
+    'groups': [
+        {
+            'tab': 'downloaders',
+            'list': 'download_providers',
+            'name': 'putio',
+            'label': 'put.io',
+            'description': 'This will start a torrent download on <a href="http://put.io">Put.io</a>.',
+            'wizard': True,
+            'options': [
+                {
+                    'name': 'enabled',
+                    'default': 0,
+                    'type': 'enabler',
+                    'radio_group': 'torrent',
+                },
+                {
+                    'name': 'oauth_token',
+                    'label': 'oauth_token',
+                    'description': 'This is the OAUTH_TOKEN from your putio API',
+                    'advanced': True,
+                },
+                {
+                    'name': 'folder',
+                    'description': ('The folder on putio where you want the upload to go','Will find the first first folder that matches this name'),
+                    'default': 0,
+                },
+                {
+                    'name': 'callback_host',
+                    'description': 'External reachable url to CP so put.io can do it\'s thing',
+                },
+                {
+                    'name': 'download',
+                    'description': 'Set this to have CouchPotato download the file from Put.io',
+                    'type': 'bool',
+                    'default': 0,
+                },
+                {
+                    'name': 'delete_file',
+                    'description': ('Set this to remove the file from putio after sucessful download','Does nothing if you don\'t select download'),
+                    'type': 'bool',
+                    'default': 0,
+                },
+                {
+                    'name': 'download_dir',
+                    'type': 'directory',
+                    'label': 'Download Directory',
+                    'description': 'The Directory to download files to, does nothing if you don\'t select download',
+                },
+                {
+                    'name': 'manual',
+                    'default': 0,
+                    'type': 'bool',
+                    'advanced': True,
+                    'description': 'Disable this downloader for automated searches, but use it when I manually send a release.',
+                },
+            ],
+        }
+    ],
+}]

couchpotato/core/downloaders/putio/main.py

@@ -0,0 +1,181 @@
+from couchpotato.api import addApiView
+from couchpotato.core.event import addEvent, fireEventAsync
+from couchpotato.core._base.downloader.main import DownloaderBase, ReleaseDownloadList
+from couchpotato.core.helpers.variable import cleanHost
+from couchpotato.core.logger import CPLog
+from couchpotato.environment import Env
+from pio import api as pio
+import datetime
+
+log = CPLog(__name__)
+
+autoload = 'Putiodownload'
+
+
+class PutIO(DownloaderBase):
+
+    protocol = ['torrent', 'torrent_magnet']
+    downloading_list = []
+    oauth_authenticate = 'https://api.couchpota.to/authorize/putio/'
+
+    def __init__(self):
+        addApiView('downloader.putio.getfrom', self.getFromPutio, docs = {
+            'desc': 'Allows you to download file from prom Put.io',
+        })
+        addApiView('downloader.putio.auth_url', self.getAuthorizationUrl)
+        addApiView('downloader.putio.credentials', self.getCredentials)
+        addEvent('putio.download', self.putioDownloader)
+
+        return super(PutIO, self).__init__()
+
+    # This is a recusive function to check for the folders
+    def recursionFolder(self, client, folder = 0, tfolder = ''):
+        files = client.File.list(folder)
+        for f in files:
+            if f.content_type == 'application/x-directory':
+                if f.name == tfolder:
+                   return f.id
+                else:
+                    result = self.recursionFolder(client, f.id, tfolder)
+                    if result != 0:
+                       return result
+        return 0
+
+    # This will check the root for the folder, and kick of recusively checking sub folder
+    def convertFolder(self, client, folder):
+        if folder == 0:
+            return 0
+        else:
+            return self.recursionFolder(client, 0, folder)
+
+    def download(self, data = None, media = None, filedata = None):
+        if not media: media = {}
+        if not data: data = {}
+
+        log.info('Sending "%s" to put.io', data.get('name'))
+        url = data.get('url')
+        client = pio.Client(self.conf('oauth_token'))
+        putioFolder = self.convertFolder(client, self.conf('folder'))
+        log.debug('putioFolder ID is %s', putioFolder)
+        # It might be possible to call getFromPutio from the renamer if we can then we don't need to do this.
+        # Note callback_host is NOT our address, it's the internet host that putio can call too
+        callbackurl = None
+        if self.conf('download'):
+            callbackurl = 'http://' + self.conf('callback_host') + '%sdownloader.putio.getfrom/' %Env.get('api_base'.strip('/'))
+        resp = client.Transfer.add_url(url, callback_url = callbackurl, parent_id = putioFolder)
+        log.debug('resp is %s', resp.id);
+        return self.downloadReturnId(resp.id)
+
+    def test(self):
+        try:
+            client = pio.Client(self.conf('oauth_token'))
+            if client.File.list():
+                return True
+        except:
+            log.info('Failed to get file listing, check OAUTH_TOKEN')
+            return False
+
+    def getAuthorizationUrl(self, host = None, **kwargs):
+
+        callback_url = cleanHost(host) + '%sdownloader.putio.credentials/' % (Env.get('api_base').lstrip('/'))
+        log.debug('callback_url is %s', callback_url)
+
+        target_url = self.oauth_authenticate + "?target=" + callback_url
+        log.debug('target_url is %s', target_url)
+
+        return {
+            'success': True,
+            'url': target_url,
+        }
+
+    def getCredentials(self, **kwargs):
+        try:
+            oauth_token = kwargs.get('oauth')
+        except:
+            return 'redirect', Env.get('web_base') + 'settings/downloaders/'
+        log.debug('oauth_token is: %s', oauth_token)
+        self.conf('oauth_token', value = oauth_token);
+        return 'redirect', Env.get('web_base') + 'settings/downloaders/'
+
+    def getAllDownloadStatus(self, ids):
+
+        log.debug('Checking putio download status.')
+        client = pio.Client(self.conf('oauth_token'))
+
+        transfers = client.Transfer.list()
+
+        log.debug(transfers);
+        release_downloads = ReleaseDownloadList(self)
+        for t in transfers:
+            if t.id in ids:
+
+                log.debug('downloading list is %s', self.downloading_list)
+                if t.status == "COMPLETED" and self.conf('download') == False :
+                    status = 'completed'
+
+                # So check if we are trying to download something
+                elif t.status == "COMPLETED" and self.conf('download') == True:
+                      # Assume we are done
+                      status = 'completed'
+                      if not self.downloading_list:
+                          now = datetime.datetime.utcnow()
+                          date_time = datetime.datetime.strptime(t.finished_at,"%Y-%m-%dT%H:%M:%S")
+                          # We need to make sure a race condition didn't happen
+                          if (now - date_time) < datetime.timedelta(minutes=5):
+                              # 5 minutes haven't passed so we wait
+                              status = 'busy'
+                      else:
+                          # If we have the file_id in the downloading_list mark it as busy
+                          if str(t.file_id) in self.downloading_list:
+                              status = 'busy'
+                else:
+                    status = 'busy'
+                release_downloads.append({
+                    'id' : t.id,
+                    'name': t.name,
+                    'status': status,
+                    'timeleft': t.estimated_time,
+                })
+
+        return release_downloads
+
+    def putioDownloader(self, fid):
+
+        log.info('Put.io Real downloader called with file_id: %s',fid)
+        client = pio.Client(self.conf('oauth_token'))
+
+        log.debug('About to get file List')
+        putioFolder = self.convertFolder(client, self.conf('folder'))
+        log.debug('PutioFolderID is %s', putioFolder)
+        files = client.File.list(parent_id=putioFolder)
+        downloaddir = self.conf('download_dir')
+
+        for f in files:
+            if str(f.id) == str(fid):
+                client.File.download(f, dest = downloaddir, delete_after_download = self.conf('delete_file'))
+                # Once the download is complete we need to remove it from the running list.
+                self.downloading_list.remove(fid)
+
+        return True
+
+    def getFromPutio(self, **kwargs):
+
+        try:
+            file_id = str(kwargs.get('file_id'))
+        except:
+            return {
+                'success' : False,
+            }
+
+        log.info('Put.io Download has been called file_id is %s', file_id)
+        if file_id not in self.downloading_list:
+            self.downloading_list.append(file_id)
+            fireEventAsync('putio.download',fid = file_id)
+            return {
+               'success': True,
+            }
+
+        return {
+            'success': False,
+        }
+

couchpotato/core/downloaders/putio/static/putio.js

@@ -0,0 +1,68 @@
+var PutIODownloader = new Class({
+
+	initialize: function(){
+		var self = this;
+
+		App.addEvent('loadSettings', self.addRegisterButton.bind(self));
+	},
+
+	addRegisterButton: function(){
+		var self = this;
+
+		var setting_page = App.getPage('Settings');
+		setting_page.addEvent('create', function(){
+
+		 	var fieldset = setting_page.tabs.downloaders.groups.putio,
+		 		l = window.location;
+
+			var putio_set = 0;
+		 	fieldset.getElements('input[type=text]').each(function(el){
+		 		putio_set += +(el.get('value') != '');
+		 	});
+
+			new Element('.ctrlHolder').adopt(
+
+			 	// Unregister button
+			 	(putio_set > 0) ?
+			 		[
+						self.unregister = new Element('a.button.red', {
+							'text': 'Unregister "'+fieldset.getElement('input[name*=oauth_token]').get('value')+'"',
+							'events': {
+								'click': function(){
+									fieldset.getElements('input[name*=oauth_token]').set('value', '').fireEvent('change');
+
+									self.unregister.destroy();
+									self.unregister_or.destroy();
+								}
+							}
+						}),
+						self.unregister_or = new Element('span[text=or]')
+					]
+			 	: null,
+
+				// Register button
+				new Element('a.button', {
+					'text': putio_set > 0 ? 'Register a different account' : 'Register your put.io account',
+					'events': {
+						'click': function(){
+							Api.request('downloader.putio.auth_url', {
+								'data': {
+									'host': l.protocol + '//' + l.hostname + (l.port ? ':' + l.port : '')
+								},
+								'onComplete': function(json){
+									window.location = json.url;
+								}
+							});
+						}
+					}
+				})
+			).inject(fieldset.getElement('.test_button'), 'before');
+		})
+
+	}
+
+});
+
+window.addEvent('domready', function(){
+	new PutIODownloader();
+});

couchpotato/core/downloaders/qbittorrent_.py

@@ -41,12 +41,30 @@ class qBittorrent(DownloaderBase):
         return self.qb
 
     def test(self):
+        """ Check if connection works
+        :return: bool
+        """
+
         if self.connect():
             return True
 
         return False
 
     def download(self, data = None, media = None, filedata = None):
+        """ Send a torrent/nzb file to the downloader
+
+        :param data: dict returned from provider
+            Contains the release information
+        :param media: media dict with information
+            Used for creating the filename when possible
+        :param filedata: downloaded torrent/nzb filedata
+            The file gets downloaded in the searcher and send to this function
+            This is done to have failed checking before using the downloader, so the downloader
+            doesn't need to worry about that
+        :return: boolean
+            One faile returns false, but the downloaded should log his own errors
+        """
+
         if not media: media = {}
         if not data: data = {}
 
@@ -95,6 +113,14 @@ class qBittorrent(DownloaderBase):
         return 'busy'
 
     def getAllDownloadStatus(self, ids):
+        """ Get status of all active downloads
+
+        :param ids: list of (mixed) downloader ids
+            Used to match the releases for this downloader as there could be
+            other downloaders active that it should ignore
+        :return: list of releases
+        """
+
         log.debug('Checking qBittorrent download status.')
 
         if not self.connect():

couchpotato/core/downloaders/rtorrent_.py

@@ -1,7 +1,7 @@
 from base64 import b16encode, b32decode
 from datetime import timedelta
 from hashlib import sha1
-from six.moves import urllib
+from urlparse import urlparse
 import os
 
 from couchpotato.core._base.downloader.main import DownloaderBase, ReleaseDownloadList
@@ -62,7 +62,7 @@ class rTorrent(DownloaderBase):
         if self.conf('ssl') and url.startswith('httprpc://'):
             url = url.replace('httprpc://', 'httprpc+https://')
 
-        parsed = urllib.urlparse(url)
+        parsed = urlparse(url)
 
         # rpc_url is only used on http/https scgi pass-through
         if parsed.scheme in ['http', 'https']:
@@ -84,6 +84,10 @@ class rTorrent(DownloaderBase):
         return self.rt
 
     def test(self):
+        """ Check if connection works
+        :return: bool
+        """
+
         if self.connect(True):
             return True
 
@@ -94,6 +98,20 @@ class rTorrent(DownloaderBase):
 
 
     def download(self, data = None, media = None, filedata = None):
+        """ Send a torrent/nzb file to the downloader
+
+        :param data: dict returned from provider
+            Contains the release information
+        :param media: media dict with information
+            Used for creating the filename when possible
+        :param filedata: downloaded torrent/nzb filedata
+            The file gets downloaded in the searcher and send to this function
+            This is done to have failed checking before using the downloader, so the downloader
+            doesn't need to worry about that
+        :return: boolean
+            One faile returns false, but the downloaded should log his own errors
+        """
+
         if not media: media = {}
         if not data: data = {}
 
@@ -161,6 +179,14 @@ class rTorrent(DownloaderBase):
         return 'completed'
 
     def getAllDownloadStatus(self, ids):
+        """ Get status of all active downloads
+
+        :param ids: list of (mixed) downloader ids
+            Used to match the releases for this downloader as there could be
+            other downloaders active that it should ignore
+        :return: list of releases
+        """
+
         log.debug('Checking rTorrent download status.')
 
         if not self.connect():

couchpotato/core/downloaders/sabnzbd.py

@@ -21,6 +21,21 @@ class Sabnzbd(DownloaderBase):
     protocol = ['nzb']
 
     def download(self, data = None, media = None, filedata = None):
+        """
+        Send a torrent/nzb file to the downloader
+
+        :param data: dict returned from provider
+            Contains the release information
+        :param media: media dict with information
+            Used for creating the filename when possible
+        :param filedata: downloaded torrent/nzb filedata
+            The file gets downloaded in the searcher and send to this function
+            This is done to have failed checking before using the downloader, so the downloader
+            doesn't need to worry about that
+        :return: boolean
+            One faile returns false, but the downloaded should log his own errors
+        """
+
         if not media: media = {}
         if not data: data = {}
 
@@ -69,6 +84,11 @@ class Sabnzbd(DownloaderBase):
             return False
 
     def test(self):
+        """ Check if connection works
+        Return message if an old version of SAB is used
+        :return: bool
+        """
+
         try:
             sab_data = self.call({
                 'mode': 'version',
@@ -89,6 +109,13 @@ class Sabnzbd(DownloaderBase):
         return True
 
     def getAllDownloadStatus(self, ids):
+        """ Get status of all active downloads
+
+        :param ids: list of (mixed) downloader ids
+            Used to match the releases for this downloader as there could be
+            other downloaders active that it should ignore
+        :return: list of releases
+        """
 
         log.debug('Checking SABnzbd download status.')
 

couchpotato/core/downloaders/synology.py

@@ -19,6 +19,21 @@ class Synology(DownloaderBase):
     status_support = False
 
     def download(self, data = None, media = None, filedata = None):
+        """
+        Send a torrent/nzb file to the downloader
+
+        :param data: dict returned from provider
+            Contains the release information
+        :param media: media dict with information
+            Used for creating the filename when possible
+        :param filedata: downloaded torrent/nzb filedata
+            The file gets downloaded in the searcher and send to this function
+            This is done to have failed checking before using the downloader, so the downloader
+            doesn't need to worry about that
+        :return: boolean
+            One faile returns false, but the downloaded should log his own errors
+        """
+
         if not media: media = {}
         if not data: data = {}
 
@@ -50,6 +65,10 @@ class Synology(DownloaderBase):
             return self.downloadReturnId('') if response else False
 
     def test(self):
+        """ Check if connection works
+        :return: bool
+        """
+
         host = cleanHost(self.conf('host'), protocol = False).split(':')
         try:
             srpc = SynologyRPC(host[0], host[1], self.conf('username'), self.conf('password'))
@@ -118,7 +137,7 @@ class SynologyRPC(object):
     def _req(self, url, args, files = None):
         response = {'success': False}
         try:
-            req = requests.post(url, data = args, files = files)
+            req = requests.post(url, data = args, files = files, verify = False)
             req.raise_for_status()
             response = json.loads(req.text)
             if response['success']:

couchpotato/core/downloaders/transmission.py

@@ -34,6 +34,21 @@ class Transmission(DownloaderBase):
         return self.trpc
 
     def download(self, data = None, media = None, filedata = None):
+        """
+        Send a torrent/nzb file to the downloader
+
+        :param data: dict returned from provider
+            Contains the release information
+        :param media: media dict with information
+            Used for creating the filename when possible
+        :param filedata: downloaded torrent/nzb filedata
+            The file gets downloaded in the searcher and send to this function
+            This is done to have failed checking before using the downloader, so the downloader
+            doesn't need to worry about that
+        :return: boolean
+            One faile returns false, but the downloaded should log his own errors
+        """
+
         if not media: media = {}
         if not data: data = {}
 
@@ -78,19 +93,32 @@ class Transmission(DownloaderBase):
             log.error('Failed sending torrent to Transmission')
             return False
 
+        data = remote_torrent.get('torrent-added') or remote_torrent.get('torrent-duplicate')
+
         # Change settings of added torrents
         if torrent_params:
-            self.trpc.set_torrent(remote_torrent['torrent-added']['hashString'], torrent_params)
+            self.trpc.set_torrent(data['hashString'], torrent_params)
 
         log.info('Torrent sent to Transmission successfully.')
-        return self.downloadReturnId(remote_torrent['torrent-added']['hashString'])
+        return self.downloadReturnId(data['hashString'])
 
     def test(self):
+        """ Check if connection works
+        :return: bool
+        """
+
         if self.connect() and self.trpc.get_session():
             return True
         return False
 
     def getAllDownloadStatus(self, ids):
+        """ Get status of all active downloads
+
+        :param ids: list of (mixed) downloader ids
+            Used to match the releases for this downloader as there could be
+            other downloaders active that it should ignore
+        :return: list of releases
+        """
 
         log.debug('Checking Transmission download status.')
 
@@ -119,6 +147,8 @@ class Transmission(DownloaderBase):
                     status = 'failed'
                 elif torrent['status'] == 0 and torrent['percentDone'] == 1:
                     status = 'completed'
+                elif torrent['status'] == 16 and torrent['percentDone'] == 1: 
+                    status = 'completed'
                 elif torrent['status'] in [5, 6]:
                     status = 'seeding'
 

couchpotato/core/downloaders/utorrent.py

@@ -51,6 +51,21 @@ class uTorrent(DownloaderBase):
         return self.utorrent_api
 
     def download(self, data = None, media = None, filedata = None):
+        """
+        Send a torrent/nzb file to the downloader
+
+        :param data: dict returned from provider
+            Contains the release information
+        :param media: media dict with information
+            Used for creating the filename when possible
+        :param filedata: downloaded torrent/nzb filedata
+            The file gets downloaded in the searcher and send to this function
+            This is done to have failed checking before using the downloader, so the downloader
+            doesn't need to worry about that
+        :return: boolean
+            One faile returns false, but the downloaded should log his own errors
+        """
+
         if not media: media = {}
         if not data: data = {}
 
@@ -120,6 +135,10 @@ class uTorrent(DownloaderBase):
         return self.downloadReturnId(torrent_hash)
 
     def test(self):
+        """ Check if connection works
+        :return: bool
+        """
+
         if self.connect():
             build_version = self.utorrent_api.get_build()
             if not build_version:
@@ -131,6 +150,13 @@ class uTorrent(DownloaderBase):
         return False
 
     def getAllDownloadStatus(self, ids):
+        """ Get status of all active downloads
+
+        :param ids: list of (mixed) downloader ids
+            Used to match the releases for this downloader as there could be
+            other downloaders active that it should ignore
+        :return: list of releases
+        """
 
         log.debug('Checking uTorrent download status.')
 

couchpotato/core/event.py

@@ -102,7 +102,7 @@ def fireEvent(name, *args, **kwargs):
             # Fire
             result = e(*args, **kwargs)
 
-        result_keys = list(result.keys())
+        result_keys = result.keys()
         result_keys.sort(key = natsortKey)
 
         if options['single'] and not options['merge']:

couchpotato/core/helpers/database.py

@@ -1,26 +0,0 @@
-
-from six import PY2
-
-
-if PY2:
-    from CodernityDB.database_super_thread_safe import SuperThreadSafeDatabase
-    from CodernityDB.index import IndexException, IndexConflict, IndexNotFoundException
-    from CodernityDB.database import RecordNotFound, RecordDeleted
-    from CodernityDB.hash_index import HashIndex
-    from CodernityDB.tree_index import MultiTreeBasedIndex, TreeBasedIndex
-else:
-    from CodernityDB3.database_super_thread_safe import SuperThreadSafeDatabase
-    from CodernityDB3.index import IndexException, IndexConflict, IndexNotFoundException
-    from CodernityDB3.database import RecordNotFound, RecordDeleted
-    from CodernityDB3.hash_index import HashIndex
-    from CodernityDB3.tree_index import MultiTreeBasedIndex, TreeBasedIndex
-
-SuperThreadSafeDatabase = SuperThreadSafeDatabase
-IndexException = IndexException
-IndexNotFoundException = IndexNotFoundException
-IndexConflict = IndexConflict
-RecordNotFound = RecordNotFound
-HashIndex = HashIndex
-MultiTreeBasedIndex = MultiTreeBasedIndex
-TreeBasedIndex = TreeBasedIndex
-RecordDeleted = RecordDeleted

couchpotato/core/helpers/encoding.py

@@ -1,11 +1,11 @@
 from string import ascii_letters, digits
+from urllib import quote_plus
 import os
 import re
 import traceback
 import unicodedata
 
 from chardet import detect
-from six.moves import urllib
 from couchpotato.core.logger import CPLog
 import six
 
@@ -16,7 +16,7 @@ log = CPLog(__name__)
 def toSafeString(original):
     valid_chars = "-_.() %s%s" % (ascii_letters, digits)
     cleaned_filename = unicodedata.normalize('NFKD', toUnicode(original)).encode('ASCII', 'ignore')
-    valid_string = ''.join(list(six.unichr(c) for c in cleaned_filename if six.unichr(c) in valid_chars))
+    valid_string = ''.join(c for c in cleaned_filename if c in valid_chars)
     return ' '.join(valid_string.split())
 
 
@@ -29,7 +29,7 @@ def simplifyString(original):
 
 def toUnicode(original, *args):
     try:
-        if isinstance(original, six.text_type):
+        if isinstance(original, unicode):
             return original
         else:
             try:
@@ -37,42 +37,29 @@ def toUnicode(original, *args):
             except:
                 try:
                     detected = detect(original)
-                    if detected.get('encoding') == 'utf-8':
+                    try:
-                        return original.decode('utf-8')
+                        if detected.get('confidence') > 0.8:
+                            return original.decode(detected.get('encoding'))
+                    except:
+                        pass
+
                     return ek(original, *args)
                 except:
                     raise
     except:
         log.error('Unable to decode value "%s..." : %s ', (repr(original)[:20], traceback.format_exc()))
-        ascii_text = str(original).encode('string_escape')
+        return 'ERROR DECODING STRING'
-        return toUnicode(ascii_text)
 
-def toUTF8(original):
-    try:
-        if isinstance(original, six.binary_type) and len(original) > 0:
-            # Try to detect
-            detected = detect(original)
-            return original.decode(detected.get('encoding')).encode('utf-8')
-        else:
-            return original
-    except:
-        #log.error('Failed encoding to UTF8: %s', traceback.format_exc())
-        raise
 
 def ss(original, *args):
 
     u_original = toUnicode(original, *args)
     try:
-        if isinstance(u_original, six.text_type):
+        from couchpotato.environment import Env
-            u_original = u_original.encode('unicode_escape')
+        return u_original.encode(Env.get('encoding'))
-        else:
-            u_original = u_original
-
-        return six.u(u_original)
     except Exception as e:
         log.debug('Failed ss encoding char, force UTF8: %s', e)
         try:
-            from couchpotato.environment import Env
             return u_original.encode(Env.get('encoding'), 'replace')
         except:
             return u_original.encode('utf-8', 'replace')
@@ -88,7 +75,7 @@ def sp(path, *args):
     if os.path.sep == '/' and '\\' in path:
         path = '/' + path.replace(':', '').replace('\\', '/')
 
-    path = os.path.normpath(path)
+    path = os.path.normpath(ss(path, *args))
 
     # Remove any trailing path separators
     if path != os.path.sep:
@@ -108,7 +95,7 @@ def ek(original, *args):
     if isinstance(original, (str, unicode)):
         try:
             from couchpotato.environment import Env
-            return original.decode(Env.get('encoding'))
+            return original.decode(Env.get('encoding'), 'ignore')
         except UnicodeDecodeError:
             raise
 
@@ -130,15 +117,14 @@ def stripAccents(s):
 def tryUrlencode(s):
     new = six.u('')
     if isinstance(s, dict):
-        for key, value in list(s.items()):
+        for key, value in s.items():
             new += six.u('&%s=%s') % (key, tryUrlencode(value))
 
         return new[1:]
     else:
         for letter in ss(s):
-            letter = six.unichr(letter)
             try:
-                new += urllib.parse.quote_plus(letter)
+                new += quote_plus(letter)
             except:
                 new += letter
 

couchpotato/core/helpers/path.py

@@ -1,51 +0,0 @@
-import os
-from chardet import detect
-from couchpotato import Env
-
-fs_enc = Env.get('fs_encoding')
-
-
-def list_dir(path, full_path = True):
-    """
-    List directory don't error when it doesn't exist
-    """
-
-    path = unicode_path(path)
-
-    if os.path.isdir(path):
-        for f in os.listdir(path):
-            if full_path:
-                yield join(path, f)
-            else:
-                yield f
-
-
-def join(*args):
-    """
-    Join path, encode properly before joining
-    """
-
-    return os.path.join(*[safe(x) for x in args])
-
-
-def unicode_path(path):
-    """
-    Convert back to unicode
-    :param path: path string
-    """
-
-    if isinstance(path, str):
-        detected = detect(path)
-        print detected
-        path = path.decode(detected.get('encoding'))
-        path = path.decode('unicode_escape')
-
-    return path
-
-
-def safe(path):
-
-    if isinstance(path, unicode):
-        return path.encode('unicode_escape')
-
-    return path

couchpotato/core/helpers/py3.py

@@ -1,2 +0,0 @@
-class NotSupported(Exception):
-    pass

couchpotato/core/helpers/request.py

@@ -1,7 +1,7 @@
+from urllib import unquote
 import re
 
 from couchpotato.core.helpers.encoding import toUnicode
-from six.moves import urllib
 from couchpotato.core.helpers.variable import natsortKey
 
 
@@ -10,7 +10,7 @@ def getParams(params):
     reg = re.compile('^[a-z0-9_\.]+$')
 
     # Sort keys
-    param_keys = list(params.keys())
+    param_keys = params.keys()
     param_keys.sort(key = natsortKey)
 
     temp = {}
@@ -28,7 +28,7 @@ def getParams(params):
 
             for item in nested:
                 if item is nested[-1]:
-                    current[item] = toUnicode(urllib.parse.unquote(value))
+                    current[item] = toUnicode(unquote(value))
                 else:
                     try:
                         current[item]
@@ -37,7 +37,7 @@ def getParams(params):
 
                     current = current[item]
         else:
-            temp[param] = toUnicode(urllib.parse.unquote(value))
+            temp[param] = toUnicode(unquote(value))
             if temp[param].lower() in ['true', 'false']:
                 temp[param] = temp[param].lower() != 'false'
 

couchpotato/core/loader.py

@@ -3,7 +3,6 @@ import sys
 import traceback
 
 from couchpotato.core.event import fireEvent
-from couchpotato.core.helpers.py3 import NotSupported
 from couchpotato.core.logger import CPLog
 from importhelper import import_module
 import six
@@ -132,7 +131,7 @@ class Loader(object):
             return False
         try:
             # Load single file plugin
-            if isinstance(module.autoload, (six.string_types, six.text_type)):
+            if isinstance(module.autoload, (str, unicode)):
                 getattr(module, module.autoload)()
             # Load folder plugin
             else:
@@ -163,8 +162,6 @@ class Loader(object):
     def loadModule(self, name):
         try:
             return import_module(name)
-        except NotSupported:
-            log.error('Module "%s" is not supported in Python 3', name)
         except ImportError:
             log.debug('Skip loading module plugin %s: %s', (name, traceback.format_exc()))
             return None

couchpotato/core/logger.py

@@ -1,6 +1,5 @@
 import logging
 import re
-import traceback
 
 
 class CPLog(object):
@@ -55,19 +54,19 @@ class CPLog(object):
 
     def safeMessage(self, msg, replace_tuple = ()):
 
-        from couchpotato.core.helpers.encoding import ss, toUTF8
+        from couchpotato.core.helpers.encoding import ss, toUnicode
 
-        msg = toUTF8(msg)
+        msg = ss(msg)
 
         try:
             if isinstance(replace_tuple, tuple):
-                msg = msg % tuple([toUTF8(x) for x in list(replace_tuple)])
+                msg = msg % tuple([ss(x) if not isinstance(x, (int, float)) else x for x in list(replace_tuple)])
             elif isinstance(replace_tuple, dict):
-                msg = msg % dict((k, toUTF8(v)) for k, v in replace_tuple.iteritems())
+                msg = msg % dict((k, ss(v) if not isinstance(v, (int, float)) else v) for k, v in replace_tuple.iteritems())
             else:
-                msg = msg % toUTF8(replace_tuple)
+                msg = msg % ss(replace_tuple)
-        except:
+        except Exception as e:
-            self.logger.error('Failed encoding stuff to log "%s": %s' % (msg, traceback.format_exc()))
+            self.logger.error('Failed encoding stuff to log "%s": %s' % (msg, e))
 
         self.setup()
         if not self.is_develop:
@@ -84,4 +83,4 @@ class CPLog(object):
             except:
                 pass
 
-        return toUTF8(msg)
+        return toUnicode(msg)

couchpotato/core/media/__init__.py

@@ -1,9 +1,10 @@
 import os
 import traceback
 
-from couchpotato import CPLog
+from couchpotato import CPLog, md5
 from couchpotato.core.event import addEvent, fireEvent, fireEventAsync
 from couchpotato.core.helpers.encoding import toUnicode
+from couchpotato.core.helpers.variable import getExt
 from couchpotato.core.plugins.base import Plugin
 import six
 
@@ -89,10 +90,18 @@ class MediaBase(Plugin):
 
         # Loop over type
         for image in image_urls.get(image_type, []):
-            if not isinstance(image, six.string_types):
+            if not isinstance(image, (str, unicode)):
                 continue
 
-            if file_type not in existing_files or len(existing_files.get(file_type, [])) == 0:
+            # Check if it has top image
+            filename = '%s.%s' % (md5(image), getExt(image))
+            existing = existing_files.get(file_type, [])
+            has_latest = False
+            for x in existing:
+                if filename in x:
+                    has_latest = True
+
+            if not has_latest or file_type not in existing_files or len(existing_files.get(file_type, [])) == 0:
                 file_path = fireEvent('file.download', url = image, single = True)
                 if file_path:
                     existing_files[file_type] = [toUnicode(file_path)]

couchpotato/core/media/_base/media/index.py

@@ -1,14 +1,14 @@
 from string import ascii_letters
 from hashlib import md5
-from couchpotato.core.helpers.database import MultiTreeBasedIndex, TreeBasedIndex
 
+from CodernityDB.tree_index import MultiTreeBasedIndex, TreeBasedIndex
 from couchpotato.core.helpers.encoding import toUnicode, simplifyString
 
 
 class MediaIndex(MultiTreeBasedIndex):
     _version = 3
 
-    custom_header = """from couchpotato.core.helpers.database import MultiTreeBasedIndex"""
+    custom_header = """from CodernityDB.tree_index import MultiTreeBasedIndex"""
 
     def __init__(self, *args, **kwargs):
         kwargs['key_format'] = '32s'
@@ -62,11 +62,10 @@ class MediaTypeIndex(TreeBasedIndex):
 
 
 class TitleSearchIndex(MultiTreeBasedIndex):
-    _version = 2
+    _version = 1
 
-    custom_header = """from couchpotato.core.helpers.database import MultiTreeBasedIndex
+    custom_header = """from CodernityDB.tree_index import MultiTreeBasedIndex
-try: from itertools import izip
+from itertools import izip
-except: izip = zip
 from couchpotato.core.helpers.encoding import simplifyString"""
 
     def __init__(self, *args, **kwargs):
@@ -102,7 +101,7 @@ from couchpotato.core.helpers.encoding import simplifyString"""
 class TitleIndex(TreeBasedIndex):
     _version = 4
 
-    custom_header = """from couchpotato.core.helpers.database import TreeBasedIndex
+    custom_header = """from CodernityDB.tree_index import TreeBasedIndex
 from string import ascii_letters
 from couchpotato.core.helpers.encoding import toUnicode, simplifyString"""
 
@@ -135,7 +134,7 @@ from couchpotato.core.helpers.encoding import toUnicode, simplifyString"""
 class StartsWithIndex(TreeBasedIndex):
     _version = 3
 
-    custom_header = """from couchpotato.core.helpers.database import TreeBasedIndex
+    custom_header = """from CodernityDB.tree_index import TreeBasedIndex
 from string import ascii_letters
 from couchpotato.core.helpers.encoding import toUnicode, simplifyString"""
 
@@ -181,7 +180,7 @@ class MediaChildrenIndex(TreeBasedIndex):
 class MediaTagIndex(MultiTreeBasedIndex):
     _version = 2
 
-    custom_header = """from couchpotato.core.helpers.database import MultiTreeBasedIndex"""
+    custom_header = """from CodernityDB.tree_index import MultiTreeBasedIndex"""
 
     def __init__(self, *args, **kwargs):
         kwargs['key_format'] = '32s'

couchpotato/core/media/_base/media/main.py

@@ -3,16 +3,15 @@ import time
 import traceback
 from string import ascii_lowercase
 
+from CodernityDB.database import RecordNotFound, RecordDeleted
 from couchpotato import tryInt, get_db
 from couchpotato.api import addApiView
 from couchpotato.core.event import fireEvent, fireEventAsync, addEvent
-from couchpotato.core.helpers.database import RecordNotFound, RecordDeleted
 from couchpotato.core.helpers.encoding import toUnicode
 from couchpotato.core.helpers.variable import splitString, getImdb, getTitle
 from couchpotato.core.logger import CPLog
 from couchpotato.core.media import MediaBase
 from .index import MediaIndex, MediaStatusIndex, MediaTypeIndex, TitleSearchIndex, TitleIndex, StartsWithIndex, MediaChildrenIndex, MediaTagIndex
-import six
 
 
 log = CPLog(__name__)
@@ -281,7 +280,7 @@ class MediaPlugin(MediaBase):
         offset = 0
         limit = -1
         if limit_offset:
-            splt = splitString(limit_offset) if isinstance(limit_offset, six.string_types) else limit_offset
+            splt = splitString(limit_offset) if isinstance(limit_offset, (str, unicode)) else limit_offset
             limit = tryInt(splt[0])
             offset = tryInt(0 if len(splt) is 1 else splt[1])
 
@@ -457,6 +456,11 @@ class MediaPlugin(MediaBase):
                         deleted = True
                     elif new_media_status:
                         media['status'] = new_media_status
+
+                        # Remove profile (no use for in manage)
+                        if new_media_status == 'done':
+                            media['profile_id'] = None
+                        
                         db.update(media)
 
                         fireEvent('media.untag', media['_id'], 'recent', single = True)
@@ -492,7 +496,7 @@ class MediaPlugin(MediaBase):
             }
         })
 
-    def restatus(self, media_id, tag_recent = True):
+    def restatus(self, media_id, tag_recent = True, allowed_restatus = None):
 
         try:
             db = get_db()
@@ -527,7 +531,7 @@ class MediaPlugin(MediaBase):
                     m['status'] = previous_status
 
             # Only update when status has changed
-            if previous_status != m['status']:
+            if previous_status != m['status'] and (not allowed_restatus or m['status'] in allowed_restatus):
                 db.update(m)
 
                 # Tag media as recent

couchpotato/core/media/_base/providers/base.py

@@ -1,4 +1,4 @@
-from six.moves import urllib
+from urlparse import urlparse
 import json
 import re
 import time
@@ -50,7 +50,7 @@ class Provider(Plugin):
         if Env.get('dev'): return True
 
         now = time.time()
-        host = urllib.urlparse(test_url).hostname
+        host = urlparse(test_url).hostname
 
         if self.last_available_check.get(host) < now - 900:
             self.last_available_check[host] = now
@@ -94,6 +94,8 @@ class Provider(Plugin):
                 try:
                     data = XMLTree.fromstring(ss(data))
                     return self.getElements(data, item_path)
+                except XMLTree.ParseError:
+                    log.error('Invalid XML returned, check "%s" manually for issues', url)
                 except:
                     log.error('Failed to parsing %s: %s', (self.getName(), traceback.format_exc()))
 
@@ -219,7 +221,7 @@ class YarrProvider(Provider):
             if provider and provider == self.getName():
                 return self
 
-            hostname = urllib.urlparse(url).hostname
+            hostname = urlparse(url).hostname
             if host and hostname in host:
                 return self
             else:

couchpotato/core/media/_base/providers/nzb/newznab.py

@@ -1,4 +1,4 @@
-from six.moves import urllib
+from urlparse import urlparse
 import time
 import traceback
 import re
@@ -68,8 +68,12 @@ class Base(NZBProvider, RSS):
             if not date:
                 date = self.getTextElement(nzb, 'pubDate')
 
-            nzb_id = self.getTextElement(nzb, 'guid').split('/')[-1:].pop()
             name = self.getTextElement(nzb, 'title')
+            detail_url = self.getTextElement(nzb, 'guid')
+            nzb_id = detail_url.split('/')[-1:].pop()
+
+            if '://' not in detail_url:
+                detail_url = (cleanHost(host['host']) + self.urls['detail']) % tryUrlencode(nzb_id)
 
             if not name:
                 continue
@@ -97,13 +101,13 @@ class Base(NZBProvider, RSS):
 
             results.append({
                 'id': nzb_id,
-                'provider_extra': urllib.urlparse(host['host']).hostname or host['host'],
+                'provider_extra': urlparse(host['host']).hostname or host['host'],
                 'name': toUnicode(name),
                 'name_extra': name_extra,
                 'age': self.calculateAge(int(time.mktime(parse(date).timetuple()))),
                 'size': int(self.getElement(nzb, 'enclosure').attrib['length']) / 1024 / 1024,
                 'url': ((self.getUrl(host['host']) + self.urls['download']) % tryUrlencode(nzb_id)) + self.getApiExt(host),
-                'detail_url': (cleanHost(host['host']) + self.urls['detail']) % tryUrlencode(nzb_id),
+                'detail_url': detail_url,
                 'content': self.getTextElement(nzb, 'description'),
                 'description': description,
                 'score': host['extra_score'],
@@ -175,7 +179,7 @@ class Base(NZBProvider, RSS):
         return '&apikey=%s' % host['api_key']
 
     def download(self, url = '', nzb_id = ''):
-        host = urllib.urlparse(url).hostname
+        host = urlparse(url).hostname
 
         if self.limits_reached.get(host):
             # Try again in 3 hours
@@ -183,7 +187,7 @@ class Base(NZBProvider, RSS):
                 return 'try_next'
 
         try:
-            data = self.urlopen(url, show_error = False)
+            data = self.urlopen(url, show_error = False, headers = {'User-Agent': Env.getIdentifier()})
             self.limits_reached[host] = False
             return data
         except HTTPError as e:

couchpotato/core/media/_base/providers/nzb/omgwtfnzbs.py

@@ -1,13 +1,9 @@
-from six.moves import urllib
-import time
-
 from couchpotato.core.event import fireEvent
 from couchpotato.core.helpers.encoding import toUnicode, tryUrlencode
 from couchpotato.core.helpers.rss import RSS
 from couchpotato.core.helpers.variable import tryInt
 from couchpotato.core.logger import CPLog
 from couchpotato.core.media._base.providers.nzb.base import NZBProvider
-from dateutil.parser import parse
 
 
 log = CPLog(__name__)
@@ -16,27 +12,19 @@ log = CPLog(__name__)
 class Base(NZBProvider, RSS):
 
     urls = {
-        'search': 'https://rss.omgwtfnzbs.org/rss-search.php?%s',
+        'search': 'https://api.omgwtfnzbs.org/json/?%s',
-        'detail_url': 'https://omgwtfnzbs.org/details.php?id=%s',
     }
 
     http_time_between_calls = 1   # Seconds
 
     cat_ids = [
-        ([15], ['dvdrip']),
+        ([15], ['dvdrip', 'scr', 'r5', 'tc', 'ts', 'cam']),
         ([15, 16], ['brrip']),
         ([16], ['720p', '1080p', 'bd50']),
         ([17], ['dvdr']),
     ]
     cat_backup_id = 'movie'
 
-    def search(self, movie, quality):
-
-        if quality['identifier'] in fireEvent('quality.pre_releases', single = True):
-            return []
-
-        return super(Base, self).search(movie, quality)
-
     def _searchOnTitle(self, title, movie, quality, results):
 
         q = '%s %s' % (title, movie['info']['year'])
@@ -47,22 +35,20 @@ class Base(NZBProvider, RSS):
             'api': self.conf('api_key', default = ''),
         })
 
-        nzbs = self.getRSSData(self.urls['search'] % params)
+        nzbs = self.getJsonData(self.urls['search'] % params)
 
-        for nzb in nzbs:
+        if isinstance(nzbs, list):
+            for nzb in nzbs:
 
-            enclosure = self.getElement(nzb, 'enclosure').attrib
+                results.append({
-            nzb_id = urllib.parse_qs(urllib.urlparse(self.getTextElement(nzb, 'link')).query).get('id')[0]
+                    'id': nzb.get('nzbid'),
-
+                    'name': toUnicode(nzb.get('release')),
-            results.append({
+                    'age': self.calculateAge(tryInt(nzb.get('usenetage'))),
-                'id': nzb_id,
+                    'size': tryInt(nzb.get('sizebytes')) / 1024 / 1024,
-                'name': toUnicode(self.getTextElement(nzb, 'title')),
+                    'url': nzb.get('getnzb'),
-                'age': self.calculateAge(int(time.mktime(parse(self.getTextElement(nzb, 'pubDate')).timetuple()))),
+                    'detail_url': nzb.get('details'),
-                'size': tryInt(enclosure['length']) / 1024 / 1024,
+                    'description': nzb.get('weblink')
-                'url': enclosure['url'],
+                })
-                'detail_url': self.urls['detail_url'] % nzb_id,
-                'description': self.getTextElement(nzb, 'description')
-            })
 
 
 config = [{

couchpotato/core/media/_base/providers/torrent/bithdtv.py

@@ -13,11 +13,11 @@ log = CPLog(__name__)
 class Base(TorrentProvider):
 
     urls = {
-        'test': 'http://www.bit-hdtv.com/',
+        'test': 'https://www.bit-hdtv.com/',
-        'login': 'http://www.bit-hdtv.com/takelogin.php',
+        'login': 'https://www.bit-hdtv.com/takelogin.php',
-        'login_check': 'http://www.bit-hdtv.com/messages.php',
+        'login_check': 'https://www.bit-hdtv.com/messages.php',
-        'detail': 'http://www.bit-hdtv.com/details.php?id=%s',
+        'detail': 'https://www.bit-hdtv.com/details.php?id=%s',
-        'search': 'http://www.bit-hdtv.com/torrents.php?',
+        'search': 'https://www.bit-hdtv.com/torrents.php?',
     }
 
     # Searches for movies only - BiT-HDTV's subcategory and resolution search filters appear to be broken
@@ -93,7 +93,7 @@ config = [{
             'tab': 'searcher',
             'list': 'torrent_providers',
             'name': 'BiT-HDTV',
-            'description': '<a href="http://bit-hdtv.com">BiT-HDTV</a>',
+            'description': '<a href="https://bit-hdtv.com">BiT-HDTV</a>',
             'wizard': True,
             'icon': 'iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAIAAACQkWg2AAAABnRSTlMAAAAAAABupgeRAAABMklEQVR4AZ3Qu0ojcQCF8W9MJcQbJNgEEQUbQVIqWgnaWfkIvoCgggixEAmIhRtY2GV3w7KwU61B0EYIxmiw0YCik84ipaCuc0nmP5dcjIUgOjqDvxf4OAdf9mnMLcUJyPyGSCP+YRdC+Kp8iagJKhuS+InYRhTGgDbeV2uEMand4ZRxizjXHQEimxhraAnUr73BNqQxMiNeV2SwcjTLEVtb4Zl10mXutvOWm2otw5Sxz6TGTbdd6ncuYvVLXAXrvM+ruyBpy1S3JLGDfUQ1O6jn5vTsrJXvqSt4UNfj6vxTRPxBHER5QeSirhLGk/5rWN+ffB1XZuxjnDy1q87m7TS+xOGA+Iv4gfkbaw+nOMXHDHnITGEk0VfRFnn4Po4vNYm6RGukmggR0L08+l+e4HMeASo/i6AJUjLgAAAAAElFTkSuQmCC',
             'options': [

couchpotato/core/media/_base/providers/torrent/hdaccess.py

@@ -0,0 +1,130 @@
+import re
+import traceback
+
+from couchpotato.core.helpers.variable import tryInt, getIdentifier
+from couchpotato.core.logger import CPLog
+from couchpotato.core.media._base.providers.torrent.base import TorrentProvider
+
+
+log = CPLog(__name__)
+
+
+class Base(TorrentProvider):
+
+    urls = {
+        'test': 'https://hdaccess.net/',
+        'detail': 'https://hdaccess.net/details.php?id=%s',
+        'search': 'https://hdaccess.net/searchapi.php?apikey=%s&username=%s&imdbid=%s&internal=%s',
+        'download': 'https://hdaccess.net/grab.php?torrent=%s&apikey=%s',
+    }
+
+    http_time_between_calls = 1  # Seconds
+
+    def _search(self, movie, quality, results):
+        data = self.getJsonData(self.urls['search'] % (self.conf('apikey'), self.conf('username'), getIdentifier(movie), self.conf('internal_only')))
+
+        if data:
+            try:
+                #for result in data[]:
+                for key, result in data.iteritems():
+                    if tryInt(result['total_results']) == 0:
+                        return
+                    torrentscore = self.conf('extra_score')
+                    releasegroup = result['releasegroup']
+                    resolution = result['resolution']
+                    encoding = result['encoding']
+                    freeleech = tryInt(result['freeleech'])
+                    seeders = tryInt(result['seeders'])
+                    torrent_desc = '/ %s / %s / %s / %s seeders' % (releasegroup, resolution, encoding, seeders)
+
+                    if freeleech > 0 and self.conf('prefer_internal'):
+                        torrent_desc += '/ Internal'
+                        torrentscore += 200
+
+                    if seeders == 0:
+                        torrentscore = 0
+
+                    name = result['release_name']
+                    year = tryInt(result['year'])
+
+                    results.append({
+                        'id': tryInt(result['torrentid']),
+                        'name': re.sub('[^A-Za-z0-9\-_ \(\).]+', '', '%s (%s) %s' % (name, year, torrent_desc)),
+                        'url': self.urls['download'] % (result['torrentid'], self.conf('apikey')),
+                        'detail_url': self.urls['detail'] % result['torrentid'],
+                        'size': tryInt(result['size']),
+                        'seeders': tryInt(result['seeders']),
+                        'leechers': tryInt(result['leechers']),
+                        'age': tryInt(result['age']),
+                        'score': torrentscore
+                    })
+            except:
+                log.error('Failed getting results from %s: %s', (self.getName(), traceback.format_exc()))
+config = [{
+    'name': 'hdaccess',
+    'groups': [
+        {
+            'tab': 'searcher',
+            'list': 'torrent_providers',
+            'name': 'HDAccess',
+            'wizard': True,
+            'description': '<a href="https://hdaccess.net">HDAccess</a>',
+            'icon': 'iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABmJLR0QA/wD/AP+gvaeTAAADuUlEQVQ4yz3T209bdQAH8O/vnNNzWno5FIpAKZdSLi23gWMDtumWuSXOyzJj9M1kyIOPS1xiYuKe9GUPezZZnGIiMTqTxS1bdIuYkG2MWKBAKYVszOgKFkrbA+259HfO+fli/PwPHzI+Pg5CCEAI2VcUlEsl1tHdU7P5bGOkWChEaaUCwvHpmkD93POn6bwgCMQGAMYYYwyCruuQnE7SPzjIstvb8l+bm5fXkokJSmlQEkUQAIpSRH5vd0tyum7I/sA1Z5VH2ctmiGWZjHw4McE1NAZtQ9fD25kXt1VN7es7dNjuGRjiJFeVpWo6slsZPhF/Ys/PPeIs2056ff7zIOS5rpU5/viJEwwEnu3Mi18dojjw0aWP6amz57h9RSE/35zinq2nuGjvIQwOj7K2SKeZWkk0auXSSZ+/ZopSy+CbW1pQKpWu6Jr2/qVPPqWRjm6HWi6Tm999g3RyGbndLCqGgVBrO3F7fHykK0YX47NNtGLYlBq/c+H2iD+3k704dHQUDcFmQVXLyP6zhfTqCl45fQYjx17FemoJunoAk1bQFGoVhkdPwNC0ix2dMT+3llodM02rKdo7gN3dHAEhuH/vNgDg3Pl3cPaNt2GZJpYX5lBbFwClBukfGobL5WrayW6NccVCISY4HIQxYts2Q3J5CXOPHuLlo6NoCoXQ2hbG0JFRpJYWcVDIQ5ZlyL5qW5b9hNlWjKsYBgzDgKppMCoGHty7A0orOHbyNNweL+obGnDm9TdhWSYS8Vn4a2shOZ0QJRGSKIHjeGGtWNhjqqpyG+k04k8eozPai9ZwByavf4kfpyZxZGwMfYOHsbwQx34hB5dL4syKweRq/xpXHwzNapqWSSYWMDszzYqFPEaOn4KiKJiZfoCZ6d8Am+GtC++iXCpjaf4P9vefT8HzfKarp3eWRKMxCILwuWXSz977YIK2RTodDoGH1+OG1+tDlbsKkuiAJEngeWBjNUUnv7rucIiOLyzTvMKJTgnVtbVXLctK3L31g+NAUajL5bEptaDpOnTdgGkzVHl9drms0ju3fnJIkphoaQtfbQiFwAcCAY5wnCE5Xff3i8XX4o9nGksH+8zl9hAGZlWMCivkc9z0L3fZ999+LTCGZKi55YJTFHfye3sc6e/vB88LpK6+iWlqSS4WcpcNXZtwOp3B6mo/REmCSSkEgd+qq3vpRkt75Fp9Y1BZWZwnhq4zEovF/u/MATAti4U7umvyu9kR27aikihC9vvTnV2xufVUMu/2uIksy/9tZvgX49fLmAMx3bsAAAAASUVORK5CYII=',
+            'options': [
+                {
+                    'name': 'enabled',
+                    'type': 'enabler',
+                    'default': False,
+                },
+                {
+                    'name': 'username',
+                    'default': '',
+                    'description': 'Enter your site username.',
+                },
+                {
+                    'name': 'apikey',
+                    'default': '',
+                    'label': 'API Key',
+                    'description': 'Enter your site api key. This can be find on <a href="https://hdaccess.net/usercp.php?action=security">Profile Security</a>',
+                },
+                {
+                    'name': 'seed_ratio',
+                    'label': 'Seed ratio',
+                    'type': 'float',
+                    'default': 0,
+                    'description': 'Will not be (re)moved until this seed ratio is met. HDAccess minimum is 1:1.',
+                },
+                {
+                    'name': 'seed_time',
+                    'label': 'Seed time',
+                    'type': 'int',
+                    'default': 0,
+                    'description': 'Will not be (re)moved until this seed time (in hours) is met. HDAccess minimum is 48 hours.',
+                },
+                {
+                    'name': 'prefer_internal',
+                    'advanced': True,
+                    'type': 'bool',
+                    'default': 1,
+                    'description': 'Favors internal releases over non-internal releases.',
+                },
+                {
+                    'name': 'internal_only',
+                    'advanced': True,
+                    'label': 'Internal Only',
+                    'type': 'bool',
+                    'default': False,
+                    'description': 'Only download releases marked as HDAccess internal',
+                },
+                {
+                    'name': 'extra_score',
+                    'advanced': True,
+                    'label': 'Extra Score',
+                    'type': 'int',
+                    'default': 0,
+                    'description': 'Starting score for each release found via this provider.',
+                }
+            ],
+        },
+    ],
+}]

couchpotato/core/media/_base/providers/torrent/hdbits.py

@@ -29,6 +29,9 @@ class Base(TorrentProvider):
         }
         post_data.update(params)
 
+        if self.conf('internal_only'):
+            post_data.update({'origin': [1]})
+
         try:
             result = self.getJsonData(self.urls['api'], data = json.dumps(post_data))
 
@@ -110,6 +113,14 @@ config = [{
                     'default': 0,
                     'description': 'Starting score for each release found via this provider.',
                 },
+                {
+                    'name': 'internal_only',
+                    'advanced': True,
+                    'label': 'Internal Only',
+                    'type': 'bool',
+                    'default': False,
+                    'description': 'Only download releases marked as HDBits internal'
+                }
             ],
         },
     ],

couchpotato/core/media/_base/providers/torrent/iptorrents.py

@@ -14,11 +14,11 @@ log = CPLog(__name__)
 class Base(TorrentProvider):
 
     urls = {
-        'test': 'https://www.iptorrents.com/',
+        'test': 'https://iptorrents.eu/',
-        'base_url': 'https://www.iptorrents.com',
+        'base_url': 'https://iptorrents.eu',
-        'login': 'https://www.iptorrents.com/torrents/',
+        'login': 'https://iptorrents.eu/torrents/',
-        'login_check': 'https://www.iptorrents.com/inbox.php',
+        'login_check': 'https://iptorrents.eu/inbox.php',
-        'search': 'https://www.iptorrents.com/torrents/?%s%%s&q=%s&qf=ti&p=%%d',
+        'search': 'https://iptorrents.eu/torrents/?%s%%s&q=%s&qf=ti&p=%%d',
     }
 
     http_time_between_calls = 1  # Seconds
@@ -120,7 +120,7 @@ config = [{
             'tab': 'searcher',
             'list': 'torrent_providers',
             'name': 'IPTorrents',
-            'description': '<a href="http://www.iptorrents.com">IPTorrents</a>',
+            'description': '<a href="https://iptorrents.eu">IPTorrents</a>',
             'wizard': True,
             'icon': 'iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAABRklEQVR42qWQO0vDUBiG8zeKY3EqQUtNO7g0J6ZJ1+ifKIIFQXAqDYKCyaaYxM3udrZLHdRFhXrZ6liCW6mubfk874EESgqaeOCF7/Y8hEh41aq6yZi2nyZgBGya9XKtZs4No05pAkZV2YbEmyMMsoSxLQeC46wCTdPPY4HruPQyGIhF97qLWsS78Miydn4XdK46NJ9OsQAYBzMIMf8MQ9wtCnTdWCaIDx/u7uljOIQEe0hiIWPamSTLay3+RxOCSPI9+RJAo7Er9r2bnqjBFAqyK+VyK4f5/Cr5ni8OFKVCz49PFI5GdNvvU7ttE1M1zMU+8AMqFksEhrMnQsBDzqmDAwzx2ehRLwT7yyCI+vSC99c3mozH1NxrJgWWtR1BOECfEJSVCm6WCzJGCA7+IWhBsM4zywDPwEp4vCjx2DzBH2ODAfsDb33Ps6dQwJgAAAAASUVORK5CYII=',
             'options': [

couchpotato/core/media/_base/providers/torrent/passthepopcorn.py

@@ -1,4 +1,4 @@
-from six.moves import html_entities
+import htmlentitydefs
 import json
 import re
 import time
@@ -145,15 +145,15 @@ class Base(TorrentProvider):
                 # character reference
                 try:
                     if txt[:3] == "&#x":
-                        return six.unichr(int(txt[3:-1], 16))
+                        return unichr(int(txt[3:-1], 16))
                     else:
-                        return six.unichr(int(txt[2:-1]))
+                        return unichr(int(txt[2:-1]))
                 except ValueError:
                     pass
             else:
                 # named entity
                 try:
-                    txt = six.unichr(html_entities.name2codepoint[txt[1:-1]])
+                    txt = unichr(htmlentitydefs.name2codepoint[txt[1:-1]])
                 except KeyError:
                     pass
             return txt  # leave as is

couchpotato/core/media/_base/providers/torrent/sceneaccess.py

@@ -42,6 +42,7 @@ class Base(TorrentProvider):
 
                     link = result.find('td', attrs = {'class': 'ttr_name'}).find('a')
                     url = result.find('td', attrs = {'class': 'td_dl'}).find('a')
+                    seeders = result.find('td', attrs = {'class': 'ttr_seeders'}).find('a')
                     leechers = result.find('td', attrs = {'class': 'ttr_leechers'}).find('a')
                     torrent_id = link['href'].replace('details?id=', '')
 
@@ -51,7 +52,7 @@ class Base(TorrentProvider):
                         'url': self.urls['download'] % url['href'],
                         'detail_url': self.urls['detail'] % torrent_id,
                         'size': self.parseSize(result.find('td', attrs = {'class': 'ttr_size'}).contents[0]),
-                        'seeders': tryInt(result.find('td', attrs = {'class': 'ttr_seeders'}).find('a').string),
+                        'seeders': tryInt(seeders.string) if seeders else 0,
                         'leechers': tryInt(leechers.string) if leechers else 0,
                         'get_more_info': self.getMoreInfo,
                     })

couchpotato/core/media/_base/providers/torrent/torrentbytes.py

@@ -1,7 +1,7 @@
 import traceback
 
 from bs4 import BeautifulSoup
-from couchpotato.core.helpers.encoding import tryUrlencode
+from couchpotato.core.helpers.encoding import tryUrlencode, toUnicode
 from couchpotato.core.helpers.variable import tryInt
 from couchpotato.core.logger import CPLog
 from couchpotato.core.media._base.providers.torrent.base import TorrentProvider
@@ -56,11 +56,12 @@ class Base(TorrentProvider):
 
                     full_id = link['href'].replace('details.php?id=', '')
                     torrent_id = full_id[:6]
+                    name = toUnicode(link.get('title', link.contents[0]).encode('ISO-8859-1')).strip()
 
                     results.append({
                         'id': torrent_id,
-                        'name': link.contents[0],
+                        'name': name,
-                        'url': self.urls['download'] % (torrent_id, link.contents[0]),
+                        'url': self.urls['download'] % (torrent_id, name),
                         'detail_url': self.urls['detail'] % torrent_id,
                         'size': self.parseSize(cells[6].contents[0] + cells[6].contents[2]),
                         'seeders': tryInt(cells[8].find('span').contents[0]),

couchpotato/core/media/_base/providers/torrent/torrentday.py

@@ -1,3 +1,4 @@
+import re
 from couchpotato.core.helpers.variable import tryInt
 from couchpotato.core.logger import CPLog
 from couchpotato.core.media._base.providers.torrent.base import TorrentProvider
@@ -8,12 +9,12 @@ log = CPLog(__name__)
 class Base(TorrentProvider):
 
     urls = {
-        'test': 'http://www.td.af/',
+        'test': 'https://torrentday.eu/',
-        'login': 'http://www.td.af/torrents/',
+        'login': 'https://torrentday.eu/torrents/',
-        'login_check': 'http://www.torrentday.com/userdetails.php',
+        'login_check': 'https://torrentday.eu/userdetails.php',
-        'detail': 'http://www.td.af/details.php?id=%s',
+        'detail': 'https://torrentday.eu/details.php?id=%s',
-        'search': 'http://www.td.af/V3/API/API.php',
+        'search': 'https://torrentday.eu/V3/API/API.php',
-        'download': 'http://www.td.af/download.php/%s/%s',
+        'download': 'https://torrentday.eu/download.php/%s/%s',
     }
 
     http_time_between_calls = 1  # Seconds
@@ -55,6 +56,10 @@ class Base(TorrentProvider):
         }
 
     def loginSuccess(self, output):
+        often = re.search('You tried too often, please wait .*</div>', output)
+        if often:
+            raise Exception(often.group(0)[:-6].strip())
+
         return 'Password not correct' not in output
 
     def loginCheckSuccess(self, output):
@@ -68,7 +73,7 @@ config = [{
             'tab': 'searcher',
             'list': 'torrent_providers',
             'name': 'TorrentDay',
-            'description': '<a href="http://www.td.af/">TorrentDay</a>',
+            'description': '<a href="https://torrentday.eu/">TorrentDay</a>',
             'wizard': True,
             'icon': 'iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAC5ElEQVQ4y12TXUgUURTH//fO7Di7foeQJH6gEEEIZZllVohfSG/6UA+RSFAQQj74VA8+Bj30lmAlRVSEvZRfhNhaka5ZUG1paKaW39tq5O6Ou+PM3M4o6m6X+XPPzD3zm/+dcy574r515WfIW8CZBM4YAA5Gc/aQC3yd7oXYEONcsISE5dTDh91HS0t7FEWhBUAeN9ynV/d9qJAgE4AECURAcVsGlCCnly26LMA0IQwTa52dje3d3e3hcPi8qqrrMjcVYI3EHCQZlkFOHBwR2QHh2ASAAIJxWGAQEDxjePhs3527XjJwnb37OHBq0T+Tyyjh+9KnEzNJ7nouc1Q/3A3HGsOvnJy+PSUlj81w2Lny9WuJ6+3AmTjD4HOcrdR2dWXLRQePvyaSLfQOPMPC8mC9iHCsOxSyzJCelzdSXlNzD5ujpb25Wbfc/XXJemTXF4+nnCNq+AMLe50uFfEJTiw4GXSFtiHL0SnIq66+p0kSArqO+eH3RdsAv9+f5vW7L7GICq6rmM8XBCAXlBw90rOyxibn5yzfkg/L09M52/jxqdESaIrBXHYZZbB1GX8cEpySxKIB8S5XcOnvqpli1zuwmrTtoLjw5LOK/eeuWsE4JH5IRPaPZKiKigmPp+5pa+u1aEjIMhEgrRkmi9mgxGUhM7LNJSzOzsE3+cOeExovXOjdytE0LV4zqNZUtV0uZzAGoGkhDH/2YHZiErmv4uyWQnZZWc+hoqL3WzlTExN5hhA8IEwkZWZOxwB++30YG/9GkYCPvqAaHAW5uWPROW86OmqCprUR7z1yZDAGQNuCvkoB/baIKUBWMTYymv+gra3eJNvjXu+B562tFyXqTJ6YuHK8rKwvBmC3vR7cOCPQLWFz8LnfXWUrJo9U19BwMyUlJRjTSMJ2ENxUiGxq9KXQfwqYlnWstvbR5aamG9g0uzM8Q4OFt++3NNixQ2NgYmeN03FOTUv7XVpV9aKisvLl1vN/WVhNc/Fi1NEAAAAASUVORK5CYII=',
             'options': [

couchpotato/core/media/_base/providers/torrent/torrentleech.py

@@ -17,7 +17,7 @@ class Base(TorrentProvider):
         'login': 'https://www.torrentleech.org/user/account/login/',
         'login_check': 'https://torrentleech.org/user/messages',
         'detail': 'https://www.torrentleech.org/torrent/%s',
-        'search': 'https://www.torrentleech.org/torrents/browse/index/query/%s/categories/%d',
+        'search': 'https://www.torrentleech.org/torrents/browse/index/query/%s/categories/%s',
         'download': 'https://www.torrentleech.org%s',
     }
 

couchpotato/core/media/_base/providers/torrent/torrentpotato.py

@@ -1,4 +1,4 @@
-from six.moves import urllib
+from urlparse import urlparse
 import re
 import traceback
 
@@ -45,7 +45,7 @@ class Base(TorrentProvider):
                         results.append({
                             'id': torrent.get('torrent_id'),
                             'protocol': 'torrent' if re.match('^(http|https|ftp)://.*$', torrent.get('download_url')) else 'torrent_magnet',
-                            'provider_extra': urllib.urlparse(host['host']).hostname or host['host'],
+                            'provider_extra': urlparse(host['host']).hostname or host['host'],
                             'name': toUnicode(torrent.get('release_name')),
                             'url': torrent.get('download_url'),
                             'detail_url': torrent.get('details_url'),

couchpotato/core/media/_base/providers/torrent/torrentshack.py

@@ -13,12 +13,12 @@ log = CPLog(__name__)
 class Base(TorrentProvider):
 
     urls = {
-        'test': 'http://torrentshack.eu/',
+        'test': 'https://theshack.us.to/',
-        'login': 'http://torrentshack.eu/login.php',
+        'login': 'https://theshack.us.to/login.php',
-        'login_check': 'http://torrentshack.eu/inbox.php',
+        'login_check': 'https://theshack.us.to/inbox.php',
-        'detail': 'http://torrentshack.eu/torrent/%s',
+        'detail': 'https://theshack.us.to/torrent/%s',
-        'search': 'http://torrentshack.eu/torrents.php?action=advanced&searchstr=%s&scene=%s&filter_cat[%d]=1',
+        'search': 'https://theshack.us.to/torrents.php?action=advanced&searchstr=%s&scene=%s&filter_cat[%d]=1',
-        'download': 'http://torrentshack.eu/%s',
+        'download': 'https://theshack.us.to/%s',
     }
 
     http_time_between_calls = 1  # Seconds
@@ -42,6 +42,7 @@ class Base(TorrentProvider):
 
                     link = result.find('span', attrs = {'class': 'torrent_name_link'}).parent
                     url = result.find('td', attrs = {'class': 'torrent_td'}).find('a')
+                    size = result.find('td', attrs = {'class': 'size'}).contents[0].strip('\n ')
                     tds = result.find_all('td')
 
                     results.append({
@@ -49,7 +50,7 @@ class Base(TorrentProvider):
                         'name': six.text_type(link.span.string).translate({ord(six.u('\xad')): None}),
                         'url': self.urls['download'] % url['href'],
                         'detail_url': self.urls['download'] % link['href'],
-                        'size': self.parseSize(result.find_all('td')[5].string),
+                        'size': self.parseSize(size),
                         'seeders': tryInt(tds[len(tds)-2].string),
                         'leechers': tryInt(tds[len(tds)-1].string),
                     })

couchpotato/core/media/_base/providers/torrent/torrentz.py

@@ -22,12 +22,12 @@ class Base(TorrentMagnetProvider, RSS):
 
     http_time_between_calls = 0
 
-    def _search(self, media, quality, results):
+    def _searchOnTitle(self, title, media, quality, results):
 
         search_url = self.urls['verified_search'] if self.conf('verified_only') else self.urls['search']
 
         # Create search parameters
-        search_params = self.buildUrl(media)
+        search_params = self.buildUrl(title, media, quality)
 
         smin = quality.get('size_min')
         smax = quality.get('size_max')

couchpotato/core/media/_base/providers/torrent/yify.py

@@ -2,28 +2,25 @@ import traceback
 
 from couchpotato.core.helpers.variable import tryInt, getIdentifier
 from couchpotato.core.logger import CPLog
-from couchpotato.core.media._base.providers.torrent.base import TorrentMagnetProvider
+from couchpotato.core.media._base.providers.torrent.base import TorrentProvider
 
 
 log = CPLog(__name__)
 
 
-class Base(TorrentMagnetProvider):
+class Base(TorrentProvider):
 
     urls = {
-        'test': '%s/api',
+        'test': '%s/api/v2',
-        'search': '%s/api/list.json?keywords=%s&quality=%s',
+        'search': '%s/api/v2/list_movies.json?limit=50&query_term=%s'
-        'detail': '%s/api/movie.json?id=%s'
     }
 
     http_time_between_calls = 1  # seconds
 
     proxy_list = [
-        'http://yify.unlocktorrent.com',
+        'https://yts.re',
-        'http://yify-torrents.com.come.in',
+        'https://yts.wf',
-        'http://yts.re',
+        'https://yts.im',
-        'http://yts.im'
-        'http://yify-torrents.im',
     ]
 
     def search(self, movie, quality):
@@ -39,28 +36,31 @@ class Base(TorrentMagnetProvider):
         if not domain:
             return
 
-        search_url = self.urls['search'] % (domain, getIdentifier(movie), quality['identifier'])
+        search_url = self.urls['search'] % (domain, getIdentifier(movie))
 
         data = self.getJsonData(search_url)
+        data = data.get('data')
 
-        if data and data.get('MovieList'):
+        if isinstance(data, dict) and data.get('movies'):
             try:
-                for result in data.get('MovieList'):
+                for result in data.get('movies'):
 
-                    if result['Quality'] and result['Quality'] not in result['MovieTitle']:
+                    for release in result.get('torrents', []):
-                        title = result['MovieTitle'] + ' BrRip ' + result['Quality']
-                    else:
-                        title = result['MovieTitle'] + ' BrRip'
 
-                    results.append({
+                        if release['quality'] and release['quality'] not in result['title_long']:
-                        'id': result['MovieID'],
+                            title = result['title_long'] + ' BRRip ' + release['quality']
-                        'name': title,
+                        else:
-                        'url': result['TorrentMagnetUrl'],
+                            title = result['title_long'] + ' BRRip'
-                        'detail_url': self.urls['detail'] % (domain, result['MovieID']),
+
-                        'size': self.parseSize(result['Size']),
+                        results.append({
-                        'seeders': tryInt(result['TorrentSeeds']),
+                            'id': release['hash'],
-                        'leechers': tryInt(result['TorrentPeers']),
+                            'name': title,
-                    })
+                            'url': release['url'],
+                            'detail_url': result['url'],
+                            'size': self.parseSize(release['size']),
+                            'seeders': tryInt(release['seeds']),
+                            'leechers': tryInt(release['peers']),
+                        })
 
             except:
                 log.error('Failed getting results from %s: %s', (self.getName(), traceback.format_exc()))

couchpotato/core/media/_base/providers/userscript/base.py

@@ -1,4 +1,4 @@
-from six.moves import urllib
+from urlparse import urlparse
 
 from couchpotato.core.event import addEvent, fireEvent
 from couchpotato.core.helpers.encoding import simplifyString
@@ -34,7 +34,7 @@ class UserscriptBase(Plugin):
 
     def belongsTo(self, url):
 
-        host = urllib.urlparse(url).hostname
+        host = urlparse(url).hostname
         host_split = host.split('.')
         if len(host_split) > 2:
             host = host[len(host_split[0]):]

couchpotato/core/media/_base/search/main.py

@@ -3,7 +3,6 @@ from couchpotato.core.event import fireEvent, addEvent
 from couchpotato.core.helpers.variable import mergeDicts, getImdb
 from couchpotato.core.logger import CPLog
 from couchpotato.core.plugins.base import Plugin
-import six
 
 log = CPLog(__name__)
 
@@ -31,7 +30,7 @@ class Search(Plugin):
     def search(self, q = '', types = None, **kwargs):
 
         # Make sure types is the correct instance
-        if isinstance(types, six.string_types):
+        if isinstance(types, (str, unicode)):
             types = [types]
         elif isinstance(types, (list, tuple, set)):
             types = list(types)

couchpotato/core/media/movie/_base/main.py

@@ -1,10 +1,10 @@
 import traceback
 import time
 
+from CodernityDB.database import RecordNotFound
 from couchpotato import get_db
 from couchpotato.api import addApiView
 from couchpotato.core.event import fireEvent, fireEventAsync, addEvent
-from couchpotato.core.helpers.database import RecordNotFound
 from couchpotato.core.helpers.encoding import toUnicode
 from couchpotato.core.helpers.variable import splitString, getTitle, getImdb, getIdentifier
 from couchpotato.core.logger import CPLog
@@ -65,7 +65,7 @@ class MovieBase(MovieTypeBase):
             return False
         elif not params.get('info'):
             try:
-                is_movie = fireEvent('movie.is_movie', identifier = params.get('identifier'), single = True)
+                is_movie = fireEvent('movie.is_movie', identifier = params.get('identifier'), adding = True, single = True)
                 if not is_movie:
                     msg = 'Can\'t add movie, seems to be a TV show.'
                     log.error(msg)

couchpotato/core/media/movie/_base/static/movie.actions.js

@@ -696,7 +696,7 @@ MA.Readd = new Class({
 
 		if(movie_done || snatched && snatched > 0)
 			self.el = new Element('a.readd', {
-				'title': 'Readd the movie and mark all previous snatched/downloaded as ignored',
+				'title': 'Re-add the movie and mark all previous snatched/downloaded as ignored',
 				'events': {
 					'click': self.doReadd.bind(self)
 				}

couchpotato/core/media/movie/charts/static/charts.css

@@ -264,3 +264,11 @@
 	height: 40px;
 }
 
+@media all and (max-width: 480px) {
+    .toggle_menu h2 {
+        font-size: 16px;
+        text-align: center;
+        height: 30px;
+    }
+}
+

couchpotato/core/media/movie/charts/static/charts.js

@@ -44,11 +44,12 @@ var Charts = new Class({
 
 		if( Cookie.read('suggestions_charts_menu_selected') === 'charts'){
 			self.show();
-			self.fireEvent.delay(0, self, 'created');
 		}
 		else
 			self.el.hide();
 
+		self.fireEvent.delay(0, self, 'created');
+
 	},
 
 	fill: function(json){

couchpotato/core/media/movie/providers/automation/crowdai.py

@@ -0,0 +1,89 @@
+import re
+
+from couchpotato.core.helpers.rss import RSS
+from couchpotato.core.helpers.variable import tryInt, splitString
+from couchpotato.core.logger import CPLog
+from couchpotato.core.media.movie.providers.automation.base import Automation
+
+
+log = CPLog(__name__)
+
+autoload = 'CrowdAI'
+
+
+class CrowdAI(Automation, RSS):
+
+    interval = 1800
+
+    def getIMDBids(self):
+
+        movies = []
+
+        urls = dict(zip(splitString(self.conf('automation_urls')), [tryInt(x) for x in splitString(self.conf('automation_urls_use'))]))
+
+        for url in urls:
+
+            if not urls[url]:
+                continue
+
+            rss_movies = self.getRSSData(url)
+
+            for movie in rss_movies:
+
+                description = self.getTextElement(movie, 'description')
+                grabs = 0
+
+                for item in movie:
+                    if item.attrib.get('name') == 'grabs':
+                        grabs = item.attrib.get('value')
+                        break
+
+                if int(grabs) > tryInt(self.conf('number_grabs')):
+                    title = re.match(r'.*Title: .a href.*/">(.*) \(\d{4}\).*', description).group(1)
+                    log.info2('%s grabs for movie: %s, enqueue...', (grabs, title))
+                    year = re.match(r'.*Year: (\d{4}).*', description).group(1)
+                    imdb = self.search(title, year)
+
+                    if imdb and self.isMinimalMovie(imdb):
+                        movies.append(imdb['imdb'])
+
+        return movies
+
+
+config = [{
+    'name': 'crowdai',
+    'groups': [
+        {
+            'tab': 'automation',
+            'list': 'automation_providers',
+            'name': 'crowdai_automation',
+            'label': 'CrowdAI',
+            'description': 'Imports from any newznab powered NZB providers RSS feed depending on the number of grabs per movie. Go to your newznab site and find the RSS section. Then copy the copy paste the link under "Movies > x264 feed" here.',
+            'options': [
+                {
+                    'name': 'automation_enabled',
+                    'default': False,
+                    'type': 'enabler',
+                },
+                {
+                    'name': 'automation_urls_use',
+                    'label': 'Use',
+                    'default': '1',
+                },
+                {
+                    'name': 'automation_urls',
+                    'label': 'url',
+                    'type': 'combined',
+                    'combine': ['automation_urls_use', 'automation_urls'],
+                    'default': 'http://YOUR_PROVIDER/rss?t=THE_MOVIE_CATEGORY&i=YOUR_USER_ID&r=YOUR_API_KEY&res=2&rls=2&num=100',
+                },
+                {
+                    'name': 'number_grabs',
+                    'default': '500',
+                    'label': 'Grab threshold',
+                    'description': 'Number of grabs required',
+                },
+            ],
+        },
+    ],
+}]

couchpotato/core/media/movie/providers/automation/letterboxd.py

@@ -48,11 +48,12 @@ class Letterboxd(Automation):
 
             soup = BeautifulSoup(self.getHTMLData(self.url % username))
 
-            for movie in soup.find_all('a', attrs = {'class': 'frame'}):
+            for movie in soup.find_all('li', attrs = {'class': 'poster-container'}):
-                match = removeEmpty(self.pattern.split(movie['title']))
+                img = movie.find('img', movie)
+                title = img.get('alt')
+
                 movies.append({
-                    'title': match[0],
+                    'title': title
-                    'year': match[1]
                 })
 
         return movies

couchpotato/core/media/movie/providers/automation/rottentomatoes.py

@@ -39,15 +39,14 @@ class Rottentomatoes(Automation, RSS):
 
                 if result:
 
-                    log.info2('Something smells...')
                     rating = tryInt(self.getTextElement(movie, rating_tag))
                     name = result.group(0)
 
+                    print rating, tryInt(self.conf('tomatometer_percent'))
                     if rating < tryInt(self.conf('tomatometer_percent')):
                         log.info2('%s seems to be rotten...', name)
                     else:
-
+                        log.info2('Found %s with fresh rating %s', (name, rating))
-                        log.info2('Found %s fresh enough movies, enqueuing: %s', (rating, name))
                         year = datetime.datetime.now().strftime("%Y")
                         imdb = self.search(name, year)
 

couchpotato/core/media/movie/providers/info/_modifier.py

@@ -1,9 +1,9 @@
 import copy
 import traceback
 
+from CodernityDB.database import RecordNotFound
 from couchpotato import get_db
 from couchpotato.core.event import addEvent, fireEvent
-from couchpotato.core.helpers.database import RecordNotFound
 from couchpotato.core.helpers.variable import mergeDicts, randomString
 from couchpotato.core.logger import CPLog
 from couchpotato.core.plugins.base import Plugin

couchpotato/core/media/movie/providers/info/couchpotatoapi.py

@@ -69,12 +69,15 @@ class CouchPotatoApi(MovieProvider):
         name_enc = base64.b64encode(ss(name))
         return self.getJsonData(self.urls['validate'] % name_enc, headers = self.getRequestHeaders())
 
-    def isMovie(self, identifier = None):
+    def isMovie(self, identifier = None, adding = False):
 
         if not identifier:
             return
 
-        data = self.getJsonData(self.urls['is_movie'] % identifier, headers = self.getRequestHeaders())
+        url = self.urls['is_movie'] % identifier
+        url += '?adding=1' if adding else ''
+
+        data = self.getJsonData(url, headers = self.getRequestHeaders())
         if data:
             return data.get('is_movie', True)
 

couchpotato/core/media/movie/providers/info/fanarttv.py

@@ -4,6 +4,7 @@ from couchpotato import tryInt
 from couchpotato.core.event import addEvent
 from couchpotato.core.logger import CPLog
 from couchpotato.core.media.movie.providers.base import MovieProvider
+from requests import HTTPError
 
 
 log = CPLog(__name__)
@@ -32,12 +33,14 @@ class FanartTV(MovieProvider):
 
         try:
             url = self.urls['api'] % identifier
-            fanart_data = self.getJsonData(url)
+            fanart_data = self.getJsonData(url, show_error = False)
 
             if fanart_data:
                 log.debug('Found images for %s', fanart_data.get('name'))
                 images = self._parseMovie(fanart_data)
-
+        except HTTPError as e:
+            log.debug('Failed getting extra art for %s: %s',
+                      (identifier, e))
         except:
             log.error('Failed getting extra art for %s: %s',
                       (identifier, traceback.format_exc()))

couchpotato/core/media/movie/providers/info/omdbapi.py

@@ -2,12 +2,12 @@ import json
 import re
 import traceback
 
+from couchpotato import Env
 from couchpotato.core.event import addEvent, fireEvent
 from couchpotato.core.helpers.encoding import tryUrlencode
 from couchpotato.core.helpers.variable import tryInt, tryFloat, splitString
 from couchpotato.core.logger import CPLog
 from couchpotato.core.media.movie.providers.base import MovieProvider
-import six
 
 
 log = CPLog(__name__)
@@ -18,8 +18,8 @@ autoload = 'OMDBAPI'
 class OMDBAPI(MovieProvider):
 
     urls = {
-        'search': 'http://www.omdbapi.com/?%s',
+        'search': 'http://www.omdbapi.com/?type=movie&%s',
-        'info': 'http://www.omdbapi.com/?i=%s',
+        'info': 'http://www.omdbapi.com/?type=movie&i=%s',
     }
 
     http_time_between_calls = 0
@@ -39,7 +39,8 @@ class OMDBAPI(MovieProvider):
             }
 
         cache_key = 'omdbapi.cache.%s' % q
-        cached = self.getCache(cache_key, self.urls['search'] % tryUrlencode({'t': name_year.get('name'), 'y': name_year.get('year', '')}), timeout = 3)
+        url = self.urls['search'] % tryUrlencode({'t': name_year.get('name'), 'y': name_year.get('year', '')})
+        cached = self.getCache(cache_key, url, timeout = 3, headers = {'User-Agent': Env.getIdentifier()})
 
         if cached:
             result = self.parseMovie(cached)
@@ -57,7 +58,7 @@ class OMDBAPI(MovieProvider):
             return {}
 
         cache_key = 'omdbapi.cache.%s' % identifier
-        cached = self.getCache(cache_key, self.urls['info'] % identifier, timeout = 3)
+        cached = self.getCache(cache_key, self.urls['info'] % identifier, timeout = 3, headers = {'User-Agent': Env.getIdentifier()})
 
         if cached:
             result = self.parseMovie(cached)
@@ -73,7 +74,7 @@ class OMDBAPI(MovieProvider):
         try:
 
             try:
-                if isinstance(movie, six.string_types):
+                if isinstance(movie, (str, unicode)):
                     movie = json.loads(movie)
             except ValueError:
                 log.info('No proper json to decode')

couchpotato/core/media/movie/providers/info/themoviedb.py

@@ -1,8 +1,7 @@
 import traceback
-import time
 
 from couchpotato.core.event import addEvent, fireEvent
-from couchpotato.core.helpers.encoding import simplifyString, toUnicode, ss, tryUrlencode
+from couchpotato.core.helpers.encoding import toUnicode, ss, tryUrlencode
 from couchpotato.core.helpers.variable import tryInt
 from couchpotato.core.logger import CPLog
 from couchpotato.core.media.movie.providers.base import MovieProvider
@@ -14,7 +13,7 @@ autoload = 'TheMovieDb'
 
 class TheMovieDb(MovieProvider):
 
-    http_time_between_calls = .3
+    http_time_between_calls = .35
 
     configuration = {
         'images': {
@@ -23,6 +22,8 @@ class TheMovieDb(MovieProvider):
     }
 
     def __init__(self):
+        addEvent('info.search', self.search, priority = 3)
+        addEvent('movie.search', self.search, priority = 3)
         addEvent('movie.info', self.getInfo, priority = 3)
         addEvent('movie.info_by_tmdb', self.getInfo)
         addEvent('app.load', self.config)
@@ -32,49 +33,45 @@ class TheMovieDb(MovieProvider):
         if configuration:
             self.configuration = configuration
 
-    def search(self, q, limit = 12):
+    def search(self, q, limit = 3):
         """ Find movie by name """
 
         if self.isDisabled():
             return False
 
-        search_string = simplifyString(q)
+        log.debug('Searching for movie: %s', q)
-        cache_key = 'tmdb.cache.%s.%s' % (search_string, limit)
-        results = None #self.getCache(cache_key)
 
-        if not results:
+        raw = None
-            log.debug('Searching for movie: %s', q)
+        try:
+            name_year = fireEvent('scanner.name_year', q, single = True)
+            raw = self.request('search/movie', {
+                'query': name_year.get('name', q),
+                'year': name_year.get('year'),
+                'search_type': 'ngram' if limit > 1 else 'phrase'
+            }, return_key = 'results')
+        except:
+            log.error('Failed searching TMDB for "%s": %s', (q, traceback.format_exc()))
 
-            raw = None
+        results = []
+        if raw:
             try:
+                nr = 0
 
-                #name_year = fireEvent('scanner.name_year', q, single = True)
+                for movie in raw:
+                    parsed_movie = self.parseMovie(movie, extended = False)
+                    if parsed_movie:
+                        results.append(parsed_movie)
 
-                raw = self.request('search/movie', {
+                    nr += 1
-                    'query': q
+                    if nr == limit:
-                }, return_key = 'results')
+                        break
-            except:
-                log.error('Failed searching TMDB for "%s": %s', (search_string, traceback.format_exc()))
 
-            results = []
+                log.info('Found: %s', [result['titles'][0] + ' (' + str(result.get('year', 0)) + ')' for result in results])
-            if raw:
-                try:
-                    nr = 0
 
-                    for movie in raw:
+                return results
-                        results.append(self.parseMovie(movie, extended = False))
+            except SyntaxError as e:
-
+                log.error('Failed to parse XML response: %s', e)
-                        nr += 1
+                return False
-                        if nr == limit:
-                            break
-
-                    log.info('Found: %s', [result['titles'][0] + ' (' + str(result.get('year', 0)) + ')' for result in results])
-
-                    self.setCache(cache_key, results)
-                    return results
-                except SyntaxError as e:
-                    log.error('Failed to parse XML response: %s', e)
-                    return False
 
         return results
 
@@ -87,93 +84,87 @@ class TheMovieDb(MovieProvider):
             'id': identifier
         }, extended = extended)
 
-        return result
+        return result or {}
 
     def parseMovie(self, movie, extended = True):
 
-        cache_key = 'tmdb.cache.%s%s' % (movie.get('id'), '.ex' if extended else '')
+        # Do request, append other items
-        movie_data = None #self.getCache(cache_key)
+        movie = self.request('movie/%s' % movie.get('id'), {
+            'append_to_response': 'alternative_titles' + (',images,casts' if extended else '')
+        })
+        if not movie:
+            return
 
-        if not movie_data:
+        # Images
+        poster = self.getImage(movie, type = 'poster', size = 'w154')
+        poster_original = self.getImage(movie, type = 'poster', size = 'original')
+        backdrop_original = self.getImage(movie, type = 'backdrop', size = 'original')
+        extra_thumbs = self.getMultImages(movie, type = 'backdrops', size = 'original') if extended else []
+
+        images = {
+            'poster': [poster] if poster else [],
+            #'backdrop': [backdrop] if backdrop else [],
+            'poster_original': [poster_original] if poster_original else [],
+            'backdrop_original': [backdrop_original] if backdrop_original else [],
+            'actors': {},
+            'extra_thumbs': extra_thumbs
+        }
+
+        # Genres
+        try:
+            genres = [genre.get('name') for genre in movie.get('genres', [])]
+        except:
+            genres = []
+
+        # 1900 is the same as None
+        year = str(movie.get('release_date') or '')[:4]
+        if not movie.get('release_date') or year == '1900' or year.lower() == 'none':
+            year = None
+
+        # Gather actors data
+        actors = {}
+        if extended:
 
             # Full data
-            movie = self.request('movie/%s' % movie.get('id'))
+            cast = movie.get('casts', {}).get('cast', [])
 
-            # Images
+            for cast_item in cast:
-            poster = self.getImage(movie, type = 'poster', size = 'w154')
+                try:
-            poster_original = self.getImage(movie, type = 'poster', size = 'original')
+                    actors[toUnicode(cast_item.get('name'))] = toUnicode(cast_item.get('character'))
-            backdrop_original = self.getImage(movie, type = 'backdrop', size = 'original')
+                    images['actors'][toUnicode(cast_item.get('name'))] = self.getImage(cast_item, type = 'profile', size = 'original')
-            extra_thumbs = self.getMultImages(movie, type = 'backdrops', size = 'original')
+                except:
+                    log.debug('Error getting cast info for %s: %s', (cast_item, traceback.format_exc()))
 
-            images = {
+        movie_data = {
-                'poster': [poster] if poster else [],
+            'type': 'movie',
-                #'backdrop': [backdrop] if backdrop else [],
+            'via_tmdb': True,
-                'poster_original': [poster_original] if poster_original else [],
+            'tmdb_id': movie.get('id'),
-                'backdrop_original': [backdrop_original] if backdrop_original else [],
+            'titles': [toUnicode(movie.get('title'))],
-                'actors': {},
+            'original_title': movie.get('original_title'),
-                'extra_thumbs': extra_thumbs
+            'images': images,
-            }
+            'imdb': movie.get('imdb_id'),
+            'runtime': movie.get('runtime'),
+            'released': str(movie.get('release_date')),
+            'year': tryInt(year, None),
+            'plot': movie.get('overview'),
+            'genres': genres,
+            'collection': getattr(movie.get('belongs_to_collection'), 'name', None),
+            'actor_roles': actors
+        }
 
-            # Genres
+        movie_data = dict((k, v) for k, v in movie_data.items() if v)
-            try:
-                genres = [genre.get('name') for genre in movie.get('genres', [])]
-            except:
-                genres = []
 
-            # 1900 is the same as None
+        # Add alternative names
-            year = str(movie.get('release_date') or '')[:4]
+        if movie_data['original_title'] and movie_data['original_title'] not in movie_data['titles']:
-            if not movie.get('release_date') or year == '1900' or year.lower() == 'none':
+            movie_data['titles'].append(movie_data['original_title'])
-                year = None
 
-            # Gather actors data
+        # Add alternative titles
-            actors = {}
+        alternate_titles = movie.get('alternative_titles', {}).get('titles', [])
-            if extended:
 
-                # Full data
+        for alt in alternate_titles:
-                cast = self.request('movie/%s/casts' % movie.get('id'), return_key = 'cast')
+            alt_name = alt.get('title')
-
+            if alt_name and alt_name not in movie_data['titles'] and alt_name.lower() != 'none' and alt_name is not None:
-                for cast_item in cast:
+                movie_data['titles'].append(alt_name)
-                    try:
-                        actors[toUnicode(cast_item.get('name'))] = toUnicode(cast_item.get('character'))
-                        images['actors'][toUnicode(cast_item.get('name'))] = self.getImage(cast_item, type = 'profile', size = 'original')
-                    except:
-                        log.debug('Error getting cast info for %s: %s', (cast_item, traceback.format_exc()))
-
-            movie_data = {
-                'type': 'movie',
-                'via_tmdb': True,
-                'tmdb_id': movie.get('id'),
-                'titles': [toUnicode(movie.get('title'))],
-                'original_title': movie.get('original_title'),
-                'images': images,
-                'imdb': movie.get('imdb_id'),
-                'runtime': movie.get('runtime'),
-                'released': str(movie.get('release_date')),
-                'year': tryInt(year, None),
-                'plot': movie.get('overview'),
-                'genres': genres,
-                'collection': getattr(movie.get('belongs_to_collection'), 'name', None),
-                'actor_roles': actors
-            }
-
-            movie_data = dict((k, v) for k, v in movie_data.items() if v)
-
-            # Add alternative names
-            if movie_data['original_title'] and movie_data['original_title'] not in movie_data['titles']:
-                movie_data['titles'].append(movie_data['original_title'])
-
-            if extended:
-
-                # Full data
-                alternate_titles = self.request('movie/%s/alternative_titles' % movie.get('id'), return_key = 'titles')
-
-                for alt in alternate_titles:
-                    alt_name = alt.get('title')
-                    if alt_name and alt_name not in movie_data['titles'] and alt_name.lower() != 'none' and alt_name is not None:
-                        movie_data['titles'].append(alt_name)
-
-            # Cache movie parsed
-            self.setCache(cache_key, movie_data)
 
         return movie_data
 
@@ -192,23 +183,26 @@ class TheMovieDb(MovieProvider):
 
         image_urls = []
         try:
-
+            for image in movie.get('images', {}).get(type, [])[1:5]:
-            # Full data
-            images = self.request('movie/%s/images' % movie.get('id'), return_key = type)
-            for image in images[1:5]:
                 image_urls.append(self.getImage(image, 'file', size))
-
         except:
             log.debug('Failed getting %s.%s for "%s"', (type, size, ss(str(movie))))
 
         return image_urls
 
     def request(self, call = '', params = {}, return_key = None):
-        params = tryUrlencode(params)
-        url = 'http://api.themoviedb.org/3/%s?api_key=%s%s' % (call, self.conf('api_key'), '&%s' % params if params else '')
-        data = self.getJsonData(url, cache_timeout = 0)
 
-        if data and return_key and data.get(return_key):
+        params = dict((k, v) for k, v in params.items() if v)
+        params = tryUrlencode(params)
+
+        try:
+            url = 'http://api.themoviedb.org/3/%s?api_key=%s%s' % (call, self.conf('api_key'), '&%s' % params if params else '')
+            data = self.getJsonData(url, show_error = False)
+        except:
+            log.debug('Movie not found: %s, %s', (call, params))
+            data = None
+
+        if data and return_key and return_key in data:
             data = data.get(return_key)
 
         return data

couchpotato/core/media/movie/providers/torrent/bitsoup.py

@@ -11,7 +11,7 @@ autoload = 'Bitsoup'
 class Bitsoup(MovieProvider, Base):
     cat_ids = [
         ([17], ['3d']),
-        ([41], ['720p', '1080p']),
+        ([80], ['720p', '1080p']),
         ([20], ['dvdr']),
         ([19], ['brrip', 'dvdrip']),
     ]

couchpotato/core/media/movie/providers/torrent/hdaccess.py

@@ -0,0 +1,11 @@
+from couchpotato.core.logger import CPLog
+from couchpotato.core.media._base.providers.torrent.hdaccess import Base
+from couchpotato.core.media.movie.providers.base import MovieProvider
+
+log = CPLog(__name__)
+
+autoload = 'HDAccess'
+
+
+class HDAccess(MovieProvider, Base):
+    pass

couchpotato/core/media/movie/providers/torrent/iptorrents.py

@@ -13,7 +13,7 @@ class IPTorrents(MovieProvider, Base):
         ([87], ['3d']),
         ([48], ['720p', '1080p', 'bd50']),
         ([72], ['cam', 'ts', 'tc', 'r5', 'scr']),
-        ([7,48], ['dvdrip', 'brrip']),
+        ([7, 48, 20], ['dvdrip', 'brrip']),
         ([6], ['dvdr']),
     ]
 

couchpotato/core/media/movie/providers/torrent/torrentleech.py

@@ -16,12 +16,12 @@ class TorrentLeech(MovieProvider, Base):
         ([9], ['ts', 'tc']),
         ([10], ['r5', 'scr']),
         ([11], ['dvdrip']),
-        ([14], ['brrip']),
+        ([13, 14], ['brrip']),
         ([12], ['dvdr']),
     ]
 
     def buildUrl(self, title, media, quality):
         return (
             tryUrlencode(title.replace(':', '')),
-            self.getCatId(quality)[0]
+            ','.join([str(x) for x in self.getCatId(quality)])
         )

couchpotato/core/media/movie/providers/torrent/torrentshack.py

@@ -22,8 +22,8 @@ class TorrentShack(MovieProvider, Base):
     #   Movies-SD Pack - 983 (not included)
 
     cat_ids = [
-        ([970], ['bd50']),
+        ([970, 320], ['bd50']),
-        ([300], ['720p', '1080p']),
+        ([300, 320], ['720p', '1080p']),
         ([350], ['dvdr']),
         ([400], ['brrip', 'dvdrip']),
     ]

couchpotato/core/media/movie/providers/torrent/torrentz.py

@@ -1,6 +1,5 @@
 from couchpotato.core.helpers.encoding import tryUrlencode
 from couchpotato.core.logger import CPLog
-from couchpotato.core.event import fireEvent
 from couchpotato.core.media._base.providers.torrent.torrentz import Base
 from couchpotato.core.media.movie.providers.base import MovieProvider
 
@@ -11,5 +10,5 @@ autoload = 'Torrentz'
 
 class Torrentz(MovieProvider, Base):
 
-    def buildUrl(self, media):
+    def buildUrl(self, title, media, quality):
-        return tryUrlencode('"%s"' % fireEvent('library.query', media, single = True))
+        return tryUrlencode('"%s %s"' % (title, media['info']['year']))

couchpotato/core/media/movie/providers/userscript/rottentomatoes.py

@@ -12,7 +12,7 @@ autoload = 'RottenTomatoes'
 
 class RottenTomatoes(UserscriptBase):
 
-    includes = ['*://www.rottentomatoes.com/m/*/']
+    includes = ['*://www.rottentomatoes.com/m/*']
     excludes = ['*://www.rottentomatoes.com/m/*/*/']
 
     version = 2

couchpotato/core/media/movie/searcher.py

@@ -166,7 +166,8 @@ class MovieSearcher(SearcherBase, MovieTypeBase):
                 'quality': q_identifier,
                 'finish': profile['finish'][index],
                 'wait_for': tryInt(profile['wait_for'][index]),
-                '3d': profile['3d'][index] if profile.get('3d') else False
+                '3d': profile['3d'][index] if profile.get('3d') else False,
+                'minimum_score': profile.get('minimum_score', 1),
             }
 
             could_not_be_released = not self.couldBeReleased(q_identifier in pre_releases, release_dates, movie['info']['year'])
@@ -202,13 +203,6 @@ class MovieSearcher(SearcherBase, MovieTypeBase):
             quality['custom'] = quality_custom
 
             results = fireEvent('searcher.search', search_protocols, movie, quality, single = True) or []
-            results_count = len(results)
-            total_result_count += results_count
-            if results_count == 0:
-                log.debug('Nothing found for %s in %s', (default_title, quality['label']))
-
-            # Keep track of releases found outside ETA window
-            outside_eta_results += results_count if could_not_be_released else 0
 
             # Check if movie isn't deleted while searching
             if not fireEvent('media.get', movie.get('_id'), single = True):
@@ -216,11 +210,17 @@ class MovieSearcher(SearcherBase, MovieTypeBase):
 
             # Add them to this movie releases list
             found_releases += fireEvent('release.create_from_search', results, movie, quality, single = True)
+            results_count = len(found_releases)
+            total_result_count += results_count
+            if results_count == 0:
+                log.debug('Nothing found for %s in %s', (default_title, quality['label']))
+
+            # Keep track of releases found outside ETA window
+            outside_eta_results += results_count if could_not_be_released else 0
 
             # Don't trigger download, but notify user of available releases
-            if could_not_be_released:
+            if could_not_be_released and results_count > 0:
-                if results_count > 0:
+                log.debug('Found %s releases for "%s", but ETA isn\'t correct yet.', (results_count, default_title))
-                    log.debug('Found %s releases for "%s", but ETA isn\'t correct yet.', (results_count, default_title))
 
             # Try find a valid result and download it
             if (force_download or not could_not_be_released or always_search) and fireEvent('release.try_download_result', results, movie, quality_custom, single = True):
@@ -394,8 +394,9 @@ class MovieSearcher(SearcherBase, MovieTypeBase):
                 log.info('Trying next release for: %s', getTitle(media))
                 self.single(media, manual = manual, force_download = force_download)
 
-            return True
+                return True
             
+            return False
         except:
             log.error('Failed searching for next release: %s', traceback.format_exc())
             return False

couchpotato/core/media/movie/suggestion/static/suggest.js

@@ -52,7 +52,7 @@ var SuggestList = new Class({
 		else
 			self.hide();
 		
-		self.fireEvent('created');
+		self.fireEvent.delay(0, self, 'created');
 
 	},
 

couchpotato/core/notifications/core/index.py

@@ -1,10 +1,10 @@
-from couchpotato.core.helpers.database import TreeBasedIndex
+from CodernityDB.tree_index import TreeBasedIndex
 
 
 class NotificationIndex(TreeBasedIndex):
     _version = 1
 
-    custom_header = """from couchpotato.core.helpers.database import TreeBasedIndex
+    custom_header = """from CodernityDB.tree_index import TreeBasedIndex
 import time"""
 
     def __init__(self, *args, **kwargs):
@@ -22,7 +22,7 @@ import time"""
 class NotificationUnreadIndex(TreeBasedIndex):
     _version = 1
 
-    custom_header = """from couchpotato.core.helpers.database import TreeBasedIndex
+    custom_header = """from CodernityDB.tree_index import TreeBasedIndex
 import time"""
 
     def __init__(self, *args, **kwargs):

couchpotato/core/notifications/core/main.py

@@ -3,17 +3,18 @@ import threading
 import time
 import traceback
 import uuid
+from CodernityDB.database import RecordDeleted
 
 from couchpotato import get_db
 from couchpotato.api import addApiView, addNonBlockApiView
 from couchpotato.core.event import addEvent, fireEvent
-from couchpotato.core.helpers.database import RecordDeleted
 from couchpotato.core.helpers.encoding import toUnicode
 from couchpotato.core.helpers.variable import tryInt, splitString
 from couchpotato.core.logger import CPLog
 from couchpotato.core.notifications.base import Notification
 from .index import NotificationIndex, NotificationUnreadIndex
 from couchpotato.environment import Env
+from tornado.ioloop import IOLoop
 
 
 log = CPLog(__name__)
@@ -110,11 +111,11 @@ class CoreNotifier(Notification):
 
         if limit_offset:
             splt = splitString(limit_offset)
-            limit = splt[0]
+            limit = tryInt(splt[0])
-            offset = 0 if len(splt) is 1 else splt[1]
+            offset = tryInt(0 if len(splt) is 1 else splt[1])
-            results = db.get_many('notification', limit = limit, offset = offset, with_doc = True)
+            results = db.all('notification', limit = limit, offset = offset, with_doc = True)
         else:
-            results = db.get_many('notification', limit = 200, with_doc = True)
+            results = db.all('notification', limit = 200, with_doc = True)
 
         notifications = []
         for n in results:
@@ -148,16 +149,15 @@ class CoreNotifier(Notification):
     def notify(self, message = '', data = None, listener = None):
         if not data: data = {}
 
+        n = {
+            '_t': 'notification',
+            'time': int(time.time()),
+        }
+
         try:
             db = get_db()
 
-            data['notification_type'] = listener if listener else 'unknown'
+            n['message'] = toUnicode(message)
-
-            n = {
-                '_t': 'notification',
-                'time': int(time.time()),
-                'message': toUnicode(message)
-            }
 
             if data.get('sticky'):
                 n['sticky'] = True
@@ -170,7 +170,7 @@ class CoreNotifier(Notification):
 
             return True
         except:
-            log.error('Failed notify: %s', traceback.format_exc())
+            log.error('Failed notify "%s": %s', (n, traceback.format_exc()))
 
     def frontend(self, type = 'notification', data = None, message = None):
         if not data: data = {}
@@ -190,7 +190,7 @@ class CoreNotifier(Notification):
         while len(self.listeners) > 0 and not self.shuttingDown():
             try:
                 listener, last_id = self.listeners.pop()
-                listener({
+                IOLoop.current().add_callback(listener, {
                     'success': True,
                     'result': [notification],
                 })

couchpotato/core/notifications/notifymywp.py

@@ -1,68 +0,0 @@
-from couchpotato.core.helpers.variable import splitString
-from couchpotato.core.logger import CPLog
-from couchpotato.core.notifications.base import Notification
-from pynmwp import PyNMWP
-import six
-
-log = CPLog(__name__)
-
-autoload = 'NotifyMyWP'
-
-
-class NotifyMyWP(Notification):
-
-    def notify(self, message = '', data = None, listener = None):
-        if not data: data = {}
-
-        keys = splitString(self.conf('api_key'))
-        p = PyNMWP(keys, self.conf('dev_key'))
-
-        response = p.push(application = self.default_title, event = message, description = message, priority = self.conf('priority'), batch_mode = len(keys) > 1)
-
-        for key in keys:
-            if not response[key]['Code'] == six.u('200'):
-                log.error('Could not send notification to NotifyMyWindowsPhone (%s). %s', (key, response[key]['message']))
-                return False
-
-        return response
-
-
-config = [{
-    'name': 'notifymywp',
-    'groups': [
-        {
-            'tab': 'notifications',
-            'list': 'notification_providers',
-            'name': 'notifymywp',
-            'label': 'Windows Phone',
-            'options': [
-                {
-                    'name': 'enabled',
-                    'default': 0,
-                    'type': 'enabler',
-                },
-                {
-                    'name': 'api_key',
-                    'description': 'Multiple keys seperated by a comma. Maximum of 5.'
-                },
-                {
-                    'name': 'dev_key',
-                    'advanced': True,
-                },
-                {
-                    'name': 'priority',
-                    'default': 0,
-                    'type': 'dropdown',
-                    'values': [('Very Low', -2), ('Moderate', -1), ('Normal', 0), ('High', 1), ('Emergency', 2)],
-                },
-                {
-                    'name': 'on_snatch',
-                    'default': 0,
-                    'type': 'bool',
-                    'advanced': True,
-                    'description': 'Also send message when movie is snatched.',
-                },
-            ],
-        }
-    ],
-}]

couchpotato/core/notifications/plex/__init__.py

@@ -23,6 +23,26 @@ config = [{
                     'default': 'localhost',
                     'description': 'Hostname/IP, default localhost'
                 },
+                {
+                    'name': 'username',
+                    'label': 'Username',
+                    'default': '',
+                    'description': 'Required for myPlex'
+                },
+                {
+                    'name': 'password',
+                    'label': 'Password',
+                    'default': '',
+                    'type': 'password',
+                    'description': 'Required for myPlex'
+                },
+                {
+                    'name': 'auth_token',
+                    'label': 'Auth Token',
+                    'default': '',
+                    'advanced': True,
+                    'description': 'Required for myPlex'
+                },
                 {
                     'name': 'clients',
                     'default': '',

couchpotato/core/notifications/plex/server.py

@@ -1,5 +1,5 @@
 from datetime import timedelta, datetime
-from six.moves import urllib
+from urlparse import urlparse
 import traceback
 
 from couchpotato.core.helpers.variable import cleanHost
@@ -35,10 +35,45 @@ class PlexServer(object):
         if path.startswith('/'):
             path = path[1:]
 
-        data = self.plex.urlopen('%s/%s' % (
+        #Maintain support for older Plex installations without myPlex
-            self.createHost(self.plex.conf('media_server'), port = 32400),
+        if not self.plex.conf('auth_token') and not self.plex.conf('username') and not self.plex.conf('password'):
-            path
+            data = self.plex.urlopen('%s/%s' % (
-        ))
+                self.createHost(self.plex.conf('media_server'), port = 32400),
+                path
+            ))
+        else:
+            #Fetch X-Plex-Token if it doesn't exist but a username/password do
+            if not self.plex.conf('auth_token') and (self.plex.conf('username') and self.plex.conf('password')):
+                import urllib2, base64
+                log.info("Fetching a new X-Plex-Token from plex.tv")
+                username = self.plex.conf('username')
+                password = self.plex.conf('password')
+                req = urllib2.Request("https://plex.tv/users/sign_in.xml", data="")
+                authheader = "Basic %s" % base64.encodestring('%s:%s' % (username, password))[:-1]
+                req.add_header("Authorization", authheader)
+                req.add_header("X-Plex-Product", "Couchpotato Notifier")
+                req.add_header("X-Plex-Client-Identifier", "b3a6b24dcab2224bdb101fc6aa08ea5e2f3147d6")
+                req.add_header("X-Plex-Version", "1.0")
+                
+                try:
+                    response = urllib2.urlopen(req)
+                except urllib2.URLError, e:
+                    log.info("Error fetching token from plex.tv")
+                
+                try:
+                    auth_tree = etree.parse(response)
+                    token = auth_tree.findall(".//authentication-token")[0].text
+                    self.plex.conf('auth_token', token)
+
+                except (ValueError, IndexError) as e:
+                    log.info("Error parsing plex.tv response: " + ex(e))
+
+            #Add X-Plex-Token header for myPlex support workaround
+            data = self.plex.urlopen('%s/%s?X-Plex-Token=%s' % (
+                self.createHost(self.plex.conf('media_server'), port = 32400),
+                path,
+                self.plex.conf('auth_token')
+            ))
         
         if data_type == 'xml':
             return etree.fromstring(data)
@@ -106,7 +141,7 @@ class PlexServer(object):
     def createHost(self, host, port = None):
 
         h = cleanHost(host)
-        p = urllib.urlparse(h)
+        p = urlparse(h)
         h = h.rstrip('/')
 
         if port and not p.port:

couchpotato/core/notifications/twitter/__init__.py

@@ -1,16 +1,8 @@
-from six import PY3
+from .main import Twitter
 
-try:
-    from .main import Twitter
 
-    def autoload():
+def autoload():
-        return Twitter()
+    return Twitter()
-except:
-    if PY3:
-        from couchpotato.core.helpers.py3 import NotSupported
-        raise NotSupported
-    else:
-        raise
 
 config = [{
     'name': 'twitter',

couchpotato/core/notifications/webhook.py

@@ -0,0 +1,68 @@
+import traceback
+
+from couchpotato.core.helpers.encoding import toUnicode
+from couchpotato.core.helpers.variable import getIdentifier
+from couchpotato.core.logger import CPLog
+from couchpotato.core.notifications.base import Notification
+
+
+log = CPLog(__name__)
+
+autoload = 'Webhook'
+
+class Webhook(Notification):
+
+    def notify(self, message = '', data = None, listener = None):
+        if not data: data = {}
+
+        post_data = {
+            'message': toUnicode(message)
+        }
+
+        if getIdentifier(data):
+            post_data.update({
+                'imdb_id': getIdentifier(data)
+            })
+
+        headers = {
+            'Content-type': 'application/x-www-form-urlencoded'
+        }
+
+        try:
+            self.urlopen(self.conf('url'), headers = headers, data = post_data, show_error = False)
+            return True
+        except:
+            log.error('Webhook notification failed: %s', traceback.format_exc())
+
+        return False
+
+
+config = [{
+    'name': 'webhook',
+    'groups': [
+        {
+            'tab': 'notifications',
+            'list': 'notification_providers',
+            'name': 'webhook',
+            'label': 'Webhook',
+            'options': [
+                {
+                    'name': 'enabled',
+                    'default': 0,
+                    'type': 'enabler',
+                },
+                {
+                    'name': 'url',
+                    'description': 'The URL to send notification data to when '
+                },
+                {
+                    'name': 'on_snatch',
+                    'default': 0,
+                    'type': 'bool',
+                    'advanced': True,
+                    'description': 'Also send message when movie is snatched.',
+                }
+            ]
+        }
+    ]
+}]

couchpotato/core/notifications/xmpp_.py

@@ -3,20 +3,12 @@ import traceback
 
 from couchpotato.core.logger import CPLog
 from couchpotato.core.notifications.base import Notification
-from six import PY3
+import xmpp
 
 
 log = CPLog(__name__)
 
-try:
+autoload = 'Xmpp'
-    import xmpp
-    autoload = 'Xmpp'
-except:
-    if PY3:
-        from couchpotato.core.helpers.py3 import NotSupported
-        raise NotSupported
-    else:
-        raise
 
 
 class Xmpp(Notification):

couchpotato/core/plugins/base.py

@@ -1,5 +1,6 @@
 import threading
-from six.moves import urllib
+from urllib import quote
+from urlparse import urlparse
 import glob
 import inspect
 import os.path
@@ -38,7 +39,7 @@ class Plugin(object):
 
     _locks = {}
 
-    user_agent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:24.0) Gecko/20130519 Firefox/24.0'
+    user_agent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:34.0) Gecko/20100101 Firefox/34.0'
     http_last_use = {}
     http_time_between_calls = 0
     http_failed_request = {}
@@ -182,13 +183,13 @@ class Plugin(object):
 
     # http request
     def urlopen(self, url, timeout = 30, data = None, headers = None, files = None, show_error = True, stream = False):
-        url = urllib.parse.quote(ss(url), safe = "%/:=&?~#+!$,;'@()*[]")
+        url = quote(ss(url), safe = "%/:=&?~#+!$,;'@()*[]")
 
         if not headers: headers = {}
         if not data: data = {}
 
         # Fill in some headers
-        parsed_url = urllib.parse.urlparse(url)
+        parsed_url = urlparse(url)
         host = '%s%s' % (parsed_url.hostname, (':' + str(parsed_url.port) if parsed_url.port else ''))
 
         headers['Referer'] = headers.get('Referer', '%s://%s' % (parsed_url.scheme, host))
@@ -205,7 +206,7 @@ class Plugin(object):
             if self.http_failed_disabled[host] > (time.time() - 900):
                 log.info2('Disabled calls to %s for 15 minutes because so many failed requests.', host)
                 if not show_error:
-                    raise Exception('Disabled calls to %s for 15 minutes because so many failed requests')
+                    raise Exception('Disabled calls to %s for 15 minutes because so many failed requests' % host)
                 else:
                     return ''
             else:
@@ -231,11 +232,7 @@ class Plugin(object):
 
             status_code = response.status_code
             if response.status_code == requests.codes.ok:
-                if stream:
+                data = response if stream else response.content
-                    data = response
-                else:
-                    data = response.content
-                    data = data.decode(response.encoding)
             else:
                 response.raise_for_status()
 
@@ -282,7 +279,7 @@ class Plugin(object):
             wait = (last_use - now) + self.http_time_between_calls
 
             if wait > 0:
-                log.debug('Waiting for %s, %d seconds', (self.getName(), wait))
+                log.debug('Waiting for %s, %d seconds', (self.getName(), max(1, wait)))
                 time.sleep(min(wait, 30))
 
     def beforeCall(self, handler):

couchpotato/core/plugins/browser.py

@@ -87,6 +87,7 @@ class FileBrowser(Plugin):
         try:
             dirs = self.getDirectories(path = path, show_hidden = show_hidden)
         except:
+            log.error('Failed getting directory "%s" : %s', (path, traceback.format_exc()))
             dirs = []
 
         parent = os.path.dirname(path.rstrip(os.path.sep))

couchpotato/core/plugins/category/index.py

@@ -1,4 +1,4 @@
-from couchpotato.core.helpers.database import TreeBasedIndex
+from CodernityDB.tree_index import TreeBasedIndex
 
 
 class CategoryIndex(TreeBasedIndex):

couchpotato/core/plugins/dashboard.py

@@ -1,14 +1,13 @@
 import random as rndm
 import time
+from CodernityDB.database import RecordDeleted
 
 from couchpotato import get_db
 from couchpotato.api import addApiView
 from couchpotato.core.event import fireEvent
-from couchpotato.core.helpers.database import RecordDeleted
 from couchpotato.core.helpers.variable import splitString, tryInt
 from couchpotato.core.logger import CPLog
 from couchpotato.core.plugins.base import Plugin
-import six
 
 
 log = CPLog(__name__)
@@ -42,7 +41,7 @@ class Dashboard(Plugin):
         # Add limit
         limit = 12
         if limit_offset:
-            splt = splitString(limit_offset) if isinstance(limit_offset, six.string_types) else limit_offset
+            splt = splitString(limit_offset) if isinstance(limit_offset, (str, unicode)) else limit_offset
             limit = tryInt(splt[0])
 
         # Get all active medias

couchpotato/core/plugins/log/main.py

@@ -1,9 +1,9 @@
-import codecs
 import os
 import re
 import traceback
 
 from couchpotato.api import addApiView
+from couchpotato.core.helpers.encoding import toUnicode
 from couchpotato.core.helpers.variable import tryInt, splitString
 from couchpotato.core.logger import CPLog
 from couchpotato.core.plugins.base import Plugin
@@ -103,8 +103,9 @@ class Logging(Plugin):
             if not os.path.isfile(path):
                 break
 
-            f = codecs.open(path, 'r', 'utf-8')
+            f = open(path, 'r')
-            raw_lines = self.toList(f.read())
+            log_content = toUnicode(f.read())
+            raw_lines = self.toList(log_content)
             raw_lines.reverse()
 
             brk = False
@@ -130,7 +131,7 @@ class Logging(Plugin):
 
     def toList(self, log_content = ''):
 
-        logs_raw = log_content.split('[0m\n')
+        logs_raw = toUnicode(log_content).split('[0m\n')
 
         logs = []
         for log_line in logs_raw:

couchpotato/core/plugins/manage.py

@@ -123,7 +123,7 @@ class Manage(Plugin):
                 fireEvent('notify.frontend', type = 'manage.update', data = True, message = 'Scanning for movies in "%s"' % folder)
 
                 onFound = self.createAddToLibrary(folder, added_identifiers)
-                fireEvent('scanner.scan', folder = folder, simple = True, newer_than = last_update if not full else 0, on_found = onFound, single = True)
+                fireEvent('scanner.scan', folder = folder, simple = True, newer_than = last_update if not full else 0, check_file_date = False, on_found = onFound, single = True)
 
                 # Break if CP wants to shut down
                 if self.shuttingDown():

couchpotato/core/plugins/profile/index.py

@@ -1,4 +1,4 @@
-from couchpotato.core.helpers.database import TreeBasedIndex
+from CodernityDB.tree_index import TreeBasedIndex
 
 
 class ProfileIndex(TreeBasedIndex):

couchpotato/core/plugins/profile/main.py

@@ -86,6 +86,7 @@ class ProfilePlugin(Plugin):
                 'label': toUnicode(kwargs.get('label')),
                 'order': tryInt(kwargs.get('order', 999)),
                 'core': kwargs.get('core', False),
+                'minimum_score': tryInt(kwargs.get('minimum_score', 1)),
                 'qualities': [],
                 'wait_for': [],
                 'stop_after': [],
@@ -217,6 +218,7 @@ class ProfilePlugin(Plugin):
                     'label': toUnicode(profile.get('label')),
                     'order': order,
                     'qualities': profile.get('qualities'),
+                    'minimum_score': 1,
                     'finish': [],
                     'wait_for': [],
                     'stop_after': [],

couchpotato/core/plugins/profile/static/profile.css

@@ -51,6 +51,11 @@
 		margin: 0 5px !important;
 	}
 
+	.profile .wait_for .minimum_score_input {
+		width: 40px !important;
+		text-align: left;
+	}
+
 	.profile .types {
 		padding: 0;
 		margin: 0 20px 0 -4px;

couchpotato/core/plugins/profile/static/profile.js

@@ -53,12 +53,21 @@ var Profile = new Class({
 				}),
 				new Element('span', {'text':'day(s) for a better quality '}),
 				new Element('span.advanced', {'text':'and keep searching'}),
+
 				// "After a checked quality is found and downloaded, continue searching for even better quality releases for the entered number of days."
 				new Element('input.inlay.xsmall.stop_after_input.advanced', {
 					'type':'text',
 					'value': data.stop_after && data.stop_after.length > 0 ? data.stop_after[0] : 0
 				}),
-				new Element('span.advanced', {'text':'day(s) for a better (checked) quality.'})
+				new Element('span.advanced', {'text':'day(s) for a better (checked) quality.'}),
+
+				// Minimum score of
+				new Element('span.advanced', {'html':'<br/>Releases need a minimum score of'}),
+				new Element('input.advanced.inlay.xsmall.minimum_score_input', {
+					'size': 4,
+					'type':'text',
+					'value': data.minimum_score || 1
+				})
 			)
 		);
 
@@ -126,6 +135,7 @@ var Profile = new Class({
 			'label' : self.el.getElement('.quality_label input').get('value'),
 			'wait_for' : self.el.getElement('.wait_for_input').get('value'),
 			'stop_after' : self.el.getElement('.stop_after_input').get('value'),
+			'minimum_score' : self.el.getElement('.minimum_score_input').get('value'),
 			'types': []
 		};
 

couchpotato/core/plugins/quality/index.py

@@ -1,5 +1,6 @@
 from hashlib import md5
-from couchpotato.core.helpers.database import HashIndex
+
+from CodernityDB.hash_index import HashIndex
 
 
 class QualityIndex(HashIndex):

couchpotato/core/plugins/quality/main.py

@@ -2,10 +2,10 @@ from math import fabs, ceil
 import traceback
 import re
 
+from CodernityDB.database import RecordNotFound
 from couchpotato import get_db
 from couchpotato.api import addApiView
 from couchpotato.core.event import addEvent, fireEvent
-from couchpotato.core.helpers.database import RecordNotFound
 from couchpotato.core.helpers.encoding import toUnicode, ss
 from couchpotato.core.helpers.variable import mergeDicts, getExt, tryInt, splitString, tryFloat
 from couchpotato.core.logger import CPLog
@@ -30,10 +30,10 @@ class QualityPlugin(Plugin):
         {'identifier': 'dvdr', 'size': (3000, 10000), 'median_size': 4500, 'label': 'DVD-R', 'alternative': ['br2dvd', ('dvd', 'r')], 'allow': [], 'ext':['iso', 'img', 'vob'], 'tags': ['pal', 'ntsc', 'video_ts', 'audio_ts', ('dvd', 'r'), 'dvd9']},
         {'identifier': 'dvdrip', 'size': (600, 2400), 'median_size': 1500, 'label': 'DVD-Rip', 'width': 720, 'alternative': [('dvd', 'rip')], 'allow': [], 'ext':['avi'], 'tags': [('dvd', 'rip'), ('dvd', 'xvid'), ('dvd', 'divx')]},
         {'identifier': 'scr', 'size': (600, 1600), 'median_size': 700, 'label': 'Screener', 'alternative': ['screener', 'dvdscr', 'ppvrip', 'dvdscreener', 'hdscr', 'webrip', ('web', 'rip')], 'allow': ['dvdr', 'dvdrip', '720p', '1080p'], 'ext':[], 'tags': []},
-        {'identifier': 'r5', 'size': (600, 1000), 'median_size': 700, 'label': 'R5', 'alternative': ['r6'], 'allow': ['dvdr', '720p'], 'ext':[]},
+        {'identifier': 'r5', 'size': (600, 1000), 'median_size': 700, 'label': 'R5', 'alternative': ['r6'], 'allow': ['dvdr', '720p', '1080p'], 'ext':[]},
-        {'identifier': 'tc', 'size': (600, 1000), 'median_size': 700, 'label': 'TeleCine', 'alternative': ['telecine'], 'allow': ['720p'], 'ext':[]},
+        {'identifier': 'tc', 'size': (600, 1000), 'median_size': 700, 'label': 'TeleCine', 'alternative': ['telecine'], 'allow': ['720p', '1080p'], 'ext':[]},
-        {'identifier': 'ts', 'size': (600, 1000), 'median_size': 700, 'label': 'TeleSync', 'alternative': ['telesync', 'hdts'], 'allow': ['720p'], 'ext':[]},
+        {'identifier': 'ts', 'size': (600, 1000), 'median_size': 700, 'label': 'TeleSync', 'alternative': ['telesync', 'hdts'], 'allow': ['720p', '1080p'], 'ext':[]},
-        {'identifier': 'cam', 'size': (600, 1000), 'median_size': 700, 'label': 'Cam', 'alternative': ['camrip', 'hdcam'], 'allow': ['720p'], 'ext':[]}
+        {'identifier': 'cam', 'size': (600, 1000), 'median_size': 700, 'label': 'Cam', 'alternative': ['camrip', 'hdcam'], 'allow': ['720p', '1080p'], 'ext':[]}
     ]
     pre_releases = ['cam', 'ts', 'tc', 'r5', 'scr']
     threed_tags = {
@@ -240,7 +240,7 @@ class QualityPlugin(Plugin):
 
         # Add additional size score if only 1 size validated
         if len(size_scores) == 1:
-            self.calcScore(score, size_scores[0], 8)
+            self.calcScore(score, size_scores[0], 7)
         del size_scores
 
         # Return nothing if all scores are <= 0
@@ -278,6 +278,8 @@ class QualityPlugin(Plugin):
             'ext': 5,
         }
 
+        scored_on = []
+
         # Check alt and tags
         for tag_type in ['identifier', 'alternative', 'tags', 'label']:
             qualities = quality.get(tag_type, [])
@@ -289,10 +291,13 @@ class QualityPlugin(Plugin):
                         log.debug('Found %s via %s %s in %s', (quality['identifier'], tag_type, quality.get(tag_type), cur_file))
                         score += points.get(tag_type)
 
-                if isinstance(alt, (str, unicode)) and ss(alt.lower()) in words:
+                if isinstance(alt, (str, unicode)) and ss(alt.lower()) in words and ss(alt.lower()) not in scored_on:
                     log.debug('Found %s via %s %s in %s', (quality['identifier'], tag_type, quality.get(tag_type), cur_file))
                     score += points.get(tag_type)
 
+                    # Don't score twice on same tag
+                    scored_on.append(ss(alt).lower())
+
         # Check extention
         for ext in quality.get('ext', []):
             if ext == extension:
@@ -485,6 +490,8 @@ class QualityPlugin(Plugin):
             'Movie Name (2015).mp4': {'size': 6500, 'quality': 'brrip'},
             'Movie Name.2014.720p Web-Dl Aac2.0 h264-ReleaseGroup': {'size': 3800, 'quality': 'brrip'},
             'Movie Name.2014.720p.WEBRip.x264.AC3-ReleaseGroup': {'size': 3000, 'quality': 'scr'},
+            'Movie.Name.2014.1080p.HDCAM.-.ReleaseGroup': {'size': 5300, 'quality': 'cam'},
+            'Movie.Name.2014.720p.HDSCR.4PARTS.MP4.AAC.ReleaseGroup': {'size': 2401, 'quality': 'scr'},
         }
 
         correct = 0

couchpotato/core/plugins/release/index.py

@@ -1,5 +1,7 @@
 from hashlib import md5
-from couchpotato.core.helpers.database import TreeBasedIndex, HashIndex
+
+from CodernityDB.hash_index import HashIndex
+from CodernityDB.tree_index import TreeBasedIndex
 
 
 class ReleaseIndex(TreeBasedIndex):

couchpotato/core/plugins/release/main.py

@@ -3,10 +3,10 @@ import os
 import time
 import traceback
 
+from CodernityDB.database import RecordDeleted, RecordNotFound
 from couchpotato import md5, get_db
 from couchpotato.api import addApiView
 from couchpotato.core.event import fireEvent, addEvent
-from couchpotato.core.helpers.database import RecordDeleted, RecordNotFound
 from couchpotato.core.helpers.encoding import toUnicode, sp
 from couchpotato.core.helpers.variable import getTitle, tryInt
 from couchpotato.core.logger import CPLog
@@ -187,7 +187,7 @@ class Release(Plugin):
             release['files'] = dict((k, [toUnicode(x) for x in v]) for k, v in group['files'].items() if v)
             db.update(release)
 
-            fireEvent('media.restatus', media['_id'], single = True)
+            fireEvent('media.restatus', media['_id'], allowed_restatus = ['done'], single = True)
 
             return True
         except:
@@ -389,8 +389,8 @@ class Release(Plugin):
                 log.info('Ignored: %s', rel['name'])
                 continue
 
-            if rel['score'] <= 0:
+            if rel['score'] < quality_custom.get('minimum_score'):
-                log.info('Ignored, score "%s" to low: %s', (rel['score'], rel['name']))
+                log.info('Ignored, score "%s" to low, need at least "%s": %s', (rel['score'], quality_custom.get('minimum_score'), rel['name']))
                 continue
 
             if rel['size'] <= 50:
@@ -441,7 +441,6 @@ class Release(Plugin):
             for rel in search_results:
 
                 rel_identifier = md5(rel['url'])
-                found_releases.append(rel_identifier)
 
                 release = {
                     '_t': 'release',
@@ -482,6 +481,9 @@ class Release(Plugin):
                 # Update release in search_results
                 rel['status'] = rls.get('status')
 
+                if rel['status'] == 'available':
+                    found_releases.append(rel_identifier)
+
             return found_releases
         except:
             log.error('Failed: %s', traceback.format_exc())

couchpotato/core/plugins/renamer.py

@@ -35,6 +35,7 @@ class Renamer(Plugin):
             'desc': 'For the renamer to check for new files to rename in a folder',
             'params': {
                 'async': {'desc': 'Optional: Set to 1 if you dont want to fire the renamer.scan asynchronous.'},
+                'to_folder': {'desc': 'Optional: The folder to move releases to. Leave empty for default folder.'},
                 'media_folder': {'desc': 'Optional: The folder of the media to scan. Keep empty for default renamer folder.'},
                 'files': {'desc': 'Optional: Provide the release files if more releases are in the same media_folder, delimited with a \'|\'. Note that no dedicated release folder is expected for releases with one file.'},
                 'base_folder': {'desc': 'Optional: The folder to find releases in. Leave empty for default folder.'},
@@ -44,6 +45,13 @@ class Renamer(Plugin):
             },
         })
 
+        addApiView('renamer.progress', self.getProgress, docs = {
+            'desc': 'Get the progress of current renamer scan',
+            'return': {'type': 'object', 'example': """{
+    'progress': False || True,
+}"""},
+        })
+
         addEvent('renamer.scan', self.scan)
         addEvent('renamer.check_snatched', self.checkSnatched)
 
@@ -67,11 +75,17 @@ class Renamer(Plugin):
 
         return True
 
+    def getProgress(self, **kwargs):
+        return {
+            'progress': self.renaming_started
+        }
+
     def scanView(self, **kwargs):
 
         async = tryInt(kwargs.get('async', 0))
         base_folder = kwargs.get('base_folder')
         media_folder = sp(kwargs.get('media_folder'))
+        to_folder = kwargs.get('to_folder')
 
         # Backwards compatibility, to be removed after a few versions :)
         if not media_folder:
@@ -95,13 +109,13 @@ class Renamer(Plugin):
                 })
 
         fire_handle = fireEvent if not async else fireEventAsync
-        fire_handle('renamer.scan', base_folder = base_folder, release_download = release_download)
+        fire_handle('renamer.scan', base_folder = base_folder, release_download = release_download, to_folder = to_folder)
 
         return {
             'success': True
         }
 
-    def scan(self, base_folder = None, release_download = None):
+    def scan(self, base_folder = None, release_download = None, to_folder = None):
         if not release_download: release_download = {}
 
         if self.isDisabled():
@@ -115,7 +129,9 @@ class Renamer(Plugin):
             base_folder = sp(self.conf('from'))
 
         from_folder = sp(self.conf('from'))
-        to_folder = sp(self.conf('to'))
+
+        if not to_folder:
+            to_folder = sp(self.conf('to'))
 
         # Get media folder to process
         media_folder = sp(release_download.get('folder'))
@@ -220,10 +236,14 @@ class Renamer(Plugin):
         nfo_name = self.conf('nfo_name')
         separator = self.conf('separator')
 
+        if len(file_name) == 0:
+            log.error('Please fill in the filename option under renamer settings. Forcing it on <original>.<ext> to keep the same name as source file.')
+            file_name = '<original>.<ext>'
+
         cd_keys = ['<cd>','<cd_nr>', '<original>']
         if not any(x in folder_name for x in cd_keys) and not any(x in file_name for x in cd_keys):
-            log.error('Missing `cd` or `cd_nr` in the renamer. This will cause multi-file releases of being renamed to the same file.'
+            log.error('Missing `cd` or `cd_nr` in the renamer. This will cause multi-file releases of being renamed to the same file. '
-                      'Force adding it')
+                      'Please add it in the renamer settings. Force adding it for now.')
             file_name = '%s %s' % ('<cd>', file_name)
 
         # Tag release folder as failed_rename in case no groups were found. This prevents check_snatched from removing the release from the downloader.
@@ -791,7 +811,7 @@ Remove it if you want it to be renamed (again, or at least let it try again)
         dest = sp(dest)
         try:
 
-            if os.path.exists(dest):
+            if os.path.exists(dest) and os.path.isfile(dest):
                 raise Exception('Destination "%s" already exists' % dest)
 
             move_type = self.conf('file_action')
@@ -865,7 +885,9 @@ Remove it if you want it to be renamed (again, or at least let it try again)
                 #If information is not available, we don't want the tag in the filename
                 replaced = replaced.replace('<' + x + '>', '')
 
-        replaced = self.replaceDoubles(replaced.lstrip('. '))
+        if self.conf('replace_doubles'):
+           replaced = self.replaceDoubles(replaced.lstrip('. '))       
+        
         for x, r in replacements.items():
             if x in ['thename', 'namethe']:
                 replaced = replaced.replace(six.u('<%s>') % toUnicode(x), toUnicode(r))
@@ -1322,6 +1344,14 @@ config = [{
                     'type': 'choice',
                     'options': rename_options
                 },
+                {
+                    'advanced': True,                
+                    'name': 'replace_doubles',
+                    'type': 'bool',
+                    'label': 'Clean Name',
+                    'description': ('Attempt to clean up double separaters due to missing data for fields.','Sometimes this eliminates wanted white space (see <a href="https://github.com/RuudBurger/CouchPotatoServer/issues/2782">#2782</a>).'),
+                    'default': True 
+                },
                 {
                     'name': 'unrar',
                     'type': 'bool',

couchpotato/core/plugins/scanner.py

@@ -131,7 +131,7 @@ class Scanner(Plugin):
         addEvent('scanner.name_year', self.getReleaseNameYear)
         addEvent('scanner.partnumber', self.getPartNumber)
 
-    def scan(self, folder = None, files = None, release_download = None, simple = False, newer_than = 0, return_ignored = True, on_found = None):
+    def scan(self, folder = None, files = None, release_download = None, simple = False, newer_than = 0, return_ignored = True, check_file_date = True, on_found = None):
 
         folder = sp(folder)
 
@@ -145,7 +145,6 @@ class Scanner(Plugin):
 
         # Scan all files of the folder if no files are set
         if not files:
-            check_file_date = True
             try:
                 files = []
                 for root, dirs, walk_files in os.walk(folder, followlinks=True):

couchpotato/core/plugins/subtitle.py

@@ -16,7 +16,7 @@ autoload = 'Subtitle'
 
 class Subtitle(Plugin):
 
-    services = ['opensubtitles', 'thesubdb', 'subswiki', 'podnapisi']
+    services = ['opensubtitles', 'thesubdb', 'subswiki', 'subscenter']
 
     def __init__(self):
         addEvent('renamer.before', self.searchSingle)

couchpotato/core/settings.py

@@ -1,14 +1,12 @@
 from __future__ import with_statement
-
+import ConfigParser
 from hashlib import md5
 
+from CodernityDB.hash_index import HashIndex
 from couchpotato.api import addApiView
 from couchpotato.core.event import addEvent, fireEvent
-from couchpotato.core.helpers.database import HashIndex
 from couchpotato.core.helpers.encoding import toUnicode
-from six.moves import configparser
 from couchpotato.core.helpers.variable import mergeDicts, tryInt, tryFloat
-import six
 
 
 class Settings(object):
@@ -64,7 +62,7 @@ class Settings(object):
     def setFile(self, config_file):
         self.file = config_file
 
-        self.p = configparser.RawConfigParser()
+        self.p = ConfigParser.RawConfigParser()
         self.p.read(config_file)
 
         from couchpotato.core.logger import CPLog
@@ -150,10 +148,7 @@ class Settings(object):
             return tryFloat(self.p.get(section, option))
 
     def getUnicode(self, section, option):
-        value = self.p.get(section, option)
+        value = self.p.get(section, option).decode('unicode_escape')
-        if six.PY2:
-            value = value.decode('unicode_escape')
-
         return toUnicode(value).strip()
 
     def getValues(self):
@@ -162,11 +157,19 @@ class Settings(object):
             values[section] = {}
             for option in self.p.items(section):
                 (option_name, option_value) = option
+
+                is_password = False
+                try: is_password = self.types[section][option_name] == 'password'
+                except: pass
+
                 values[section][option_name] = self.get(option_name, section)
+                if is_password and values[section][option_name]:
+                    values[section][option_name] = len(values[section][option_name]) * '*'
+
         return values
 
     def save(self):
-        with open(self.file, 'w') as configfile:
+        with open(self.file, 'wb') as configfile:
             self.p.write(configfile)
 
         self.log.debug('Saved settings')

couchpotato/environment.py

@@ -14,7 +14,6 @@ class Env(object):
     ''' Environment variables '''
     _app = None
     _encoding = 'UTF-8'
-    _fs_encoding = 'UTF-8'
     _debug = False
     _dev = False
     _settings = Settings()

couchpotato/runner.py

@@ -11,12 +11,12 @@ import re
 import tarfile
 import shutil
 
+from CodernityDB.database_super_thread_safe import SuperThreadSafeDatabase
 from argparse import ArgumentParser
 from cache import FileSystemCache
 from couchpotato import KeyHandler, LoginHandler, LogoutHandler
 from couchpotato.api import NonBlockHandler, ApiHandler
 from couchpotato.core.event import fireEventAsync, fireEvent
-from couchpotato.core.helpers.database import SuperThreadSafeDatabase
 from couchpotato.core.helpers.encoding import sp
 from couchpotato.core.helpers.variable import getDataDir, tryInt, getFreeSpace
 import requests
@@ -86,7 +86,6 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En
         encoding = 'UTF-8'
 
     Env.set('encoding', encoding)
-    Env.set('fs_encoding', sys.getfilesystemencoding())
 
     # Do db stuff
     db_path = sp(os.path.join(data_dir, 'database'))
@@ -117,7 +116,8 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En
 
                     # Delete non zip files
                     if len(ints) != 1:
-                        os.remove(os.path.join(root, backup_file))
+                        try: os.remove(os.path.join(root, backup_file))
+                        except: pass
                     else:
                         existing_backups.append((int(ints[0]), backup_file))
             else:
@@ -205,7 +205,7 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En
         logger.addHandler(hdlr)
 
     # To file
-    hdlr2 = handlers.RotatingFileHandler(Env.get('log_path'), 'a', 500000, 10, encoding = 'utf-8')
+    hdlr2 = handlers.RotatingFileHandler(Env.get('log_path'), 'a', 500000, 10, encoding = Env.get('encoding'))
     hdlr2.setFormatter(formatter)
     logger.addHandler(hdlr2)
 
@@ -244,11 +244,13 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En
 
     # Basic config
     host = Env.setting('host', default = '0.0.0.0')
-    # app.debug = development
+    host6 = Env.setting('host6', default = '::')
+
     config = {
         'use_reloader': reloader,
         'port': tryInt(Env.setting('port', default = 5050)),
         'host': host if host and len(host) > 0 else '0.0.0.0',
+        'host6': host6 if host6 and len(host6) > 0 else '::',
         'ssl_cert': Env.setting('ssl_cert', default = None),
         'ssl_key': Env.setting('ssl_key', default = None),
     }
@@ -331,6 +333,10 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En
     while try_restart:
         try:
             server.listen(config['port'], config['host'])
+            
+            try: server.listen(config['port'], config['host6'])
+            except: log.info2('Tried to bind to IPV6 but failed')
+
             loop.start()
             server.close_all_connections()
             server.stop()

couchpotato/static/scripts/couchpotato.js

@@ -54,16 +54,22 @@
 	},
 
 	pushState: function(e){
-		if((!e.meta && Browser.platform.mac) || (!e.control && !Browser.platform.mac)){
+		var self = this;
+
+		if((!e.meta && self.isMac()) || (!e.control && !self.isMac())){
 			(e).preventDefault();
 			var url = e.target.get('href');
-			if(History.getPath() != url)
+
+			// Middle click
+			if(e.event && e.event.button == 1)
+				window.open(url);
+			else if(History.getPath() != url)
 				History.push(url);
 		}
 	},
 
 	isMac: function(){
-		return Browser.platform.mac
+		return Browser.platform == 'mac'
 	},
 
 	createLayout: function(){
@@ -325,11 +331,12 @@
 	},
 
 	openDerefered: function(e, el){
+		var self = this;
 		(e).stop();
 
 		var url = 'http://www.dereferer.org/?' + el.get('href');
 
-		if(el.get('target') == '_blank' || (e.meta && Browser.platform.mac) || (e.control && !Browser.platform.mac))
+		if(el.get('target') == '_blank' || (e.meta && self.isMac()) || (e.control && !self.isMac()))
 			window.open(url);
 		else
 			window.location = url;