Merge branch 'refs/heads/develop' into desktop

couchpotato/__init__.py

@@ -1,3 +1,7 @@
+import os
+import time
+import traceback
+
 from couchpotato.api import api_docs, api_docs_missing, api
 from couchpotato.core.event import fireEvent
 from couchpotato.core.helpers.variable import md5, tryInt
@@ -5,9 +9,6 @@ from couchpotato.core.logger import CPLog
 from couchpotato.environment import Env
 from tornado import template
 from tornado.web import RequestHandler, authenticated
-import os
-import time
-import traceback
 
 
 log = CPLog(__name__)

couchpotato/api.py

@@ -89,6 +89,7 @@ class ApiHandler(RequestHandler):
         route = route.strip('/')
         if not api.get(route):
             self.write('API call doesn\'t seem to exist')
+            self.finish()
             return
 
         # Create lock if it doesn't exist

couchpotato/core/_base/downloader/main.py

@@ -25,6 +25,7 @@ class DownloaderBase(Provider):
     status_support = True
 
     torrent_sources = [
+        'https://zoink.it/torrent/%s.torrent',
         'http://torrage.com/torrent/%s.torrent',
         'https://torcache.net/torrent/%s.torrent',
     ]

couchpotato/core/database.py

@@ -3,10 +3,11 @@ import os
 import time
 import traceback
 
+from CodernityDB.index import IndexException, IndexNotFoundException, IndexConflict
 from couchpotato import CPLog
 from couchpotato.api import addApiView
 from couchpotato.core.event import addEvent, fireEvent
-from couchpotato.core.helpers.encoding import toUnicode
+from couchpotato.core.helpers.encoding import toUnicode, sp
 from couchpotato.core.helpers.variable import getImdb, tryInt
 
 
@@ -15,11 +16,13 @@ log = CPLog(__name__)
 
 class Database(object):
 
-    indexes = []
+    indexes = None
     db = None
 
     def __init__(self):
 
+        self.indexes = {}
+
         addApiView('database.list_documents', self.listDocuments)
         addApiView('database.reindex', self.reindex)
         addApiView('database.compact', self.compact)
@@ -45,26 +48,45 @@ class Database(object):
 
     def setupIndex(self, index_name, klass):
 
-        self.indexes.append(index_name)
+        self.indexes[index_name] = klass
 
         db = self.getDB()
 
         # Category index
         index_instance = klass(db.path, index_name)
         try:
-            db.add_index(index_instance)
-            db.reindex_index(index_name)
-        except:
-            previous = db.indexes_names[index_name]
-            previous_version = previous._version
-            current_version = klass._version
 
-            # Only edit index if versions are different
+            # Make sure store and bucket don't exist
-            if previous_version < current_version:
+            exists = []
-                log.debug('Index "%s" already exists, updating and reindexing', index_name)
+            for x in ['buck', 'stor']:
-                db.destroy_index(previous)
+                full_path = os.path.join(db.path, '%s_%s' % (index_name, x))
+                if os.path.exists(full_path):
+                    exists.append(full_path)
+
+            if index_name not in db.indexes_names:
+
+                # Remove existing buckets if index isn't there
+                for x in exists:
+                    os.unlink(x)
+
+                # Add index (will restore buckets)
                 db.add_index(index_instance)
                 db.reindex_index(index_name)
+            else:
+                # Previous info
+                previous = db.indexes_names[index_name]
+                previous_version = previous._version
+                current_version = klass._version
+
+                # Only edit index if versions are different
+                if previous_version < current_version:
+                    log.debug('Index "%s" already exists, updating and reindexing', index_name)
+                    db.destroy_index(previous)
+                    db.add_index(index_instance)
+                    db.reindex_index(index_name)
+
+        except:
+            log.error('Failed adding index %s: %s', (index_name, traceback.format_exc()))
 
     def deleteDocument(self, **kwargs):
 
@@ -138,21 +160,62 @@ class Database(object):
             'success': success
         }
 
-    def compact(self, **kwargs):
+    def compact(self, try_repair = True, **kwargs):
+
+        success = False
+        db = self.getDB()
+
+        # Removing left over compact files
+        db_path = sp(db.path)
+        for f in os.listdir(sp(db.path)):
+            for x in ['_compact_buck', '_compact_stor']:
+                if f[-len(x):] == x:
+                    os.unlink(os.path.join(db_path, f))
 
-        success = True
         try:
             start = time.time()
-            db = self.getDB()
             size = float(db.get_db_details().get('size', 0))
             log.debug('Compacting database, current size: %sMB', round(size/1048576, 2))
 
             db.compact()
             new_size = float(db.get_db_details().get('size', 0))
             log.debug('Done compacting database in %ss, new size: %sMB, saved: %sMB', (round(time.time()-start, 2), round(new_size/1048576, 2), round((size-new_size)/1048576, 2)))
+            success = True
+        except (IndexException, AttributeError):
+            if try_repair:
+                log.error('Something wrong with indexes, trying repair')
+
+                # Remove all indexes
+                old_indexes = self.indexes.keys()
+                for index_name in old_indexes:
+                    try:
+                        db.destroy_index(index_name)
+                    except IndexNotFoundException:
+                        pass
+                    except:
+                        log.error('Failed removing old index %s', index_name)
+
+                # Add them again
+                for index_name in self.indexes:
+                    klass = self.indexes[index_name]
+
+                    # Category index
+                    index_instance = klass(db.path, index_name)
+                    try:
+                        db.add_index(index_instance)
+                        db.reindex_index(index_name)
+                    except IndexConflict:
+                        pass
+                    except:
+                        log.error('Failed adding index %s', index_name)
+                        raise
+
+                self.compact(try_repair = False)
+            else:
+                log.error('Failed compact: %s', traceback.format_exc())
+
         except:
             log.error('Failed compact: %s', traceback.format_exc())
-            success = False
 
         return {
             'success': success
@@ -166,6 +229,7 @@ class Database(object):
         size = db.get_db_details().get('size')
         prop_name = 'last_db_compact'
         last_check = int(Env.prop(prop_name, default = 0))
+
         if size > 26214400 and last_check < time.time()-604800: # 25MB / 7 days
             self.compact()
             Env.prop(prop_name, value = int(time.time()))

couchpotato/core/downloaders/rtorrent_.py

@@ -5,7 +5,6 @@ from urlparse import urlparse
 import os
 
 from couchpotato.core._base.downloader.main import DownloaderBase, ReleaseDownloadList
-
 from couchpotato.core.event import addEvent
 from couchpotato.core.helpers.encoding import sp
 from couchpotato.core.helpers.variable import cleanHost, splitString

couchpotato/core/helpers/variable.py

@@ -1,4 +1,5 @@
 import collections
+import ctypes
 import hashlib
 import os
 import platform
@@ -291,9 +292,14 @@ def dictIsSubset(a, b):
     return all([k in b and b[k] == v for k, v in a.items()])
 
 
+# Returns True if sub_folder is the same as or inside base_folder
 def isSubFolder(sub_folder, base_folder):
-    # Returns True if sub_folder is the same as or inside base_folder
+    if base_folder and sub_folder:
-    return base_folder and sub_folder and ss(os.path.normpath(base_folder).rstrip(os.path.sep) + os.path.sep) in ss(os.path.normpath(sub_folder).rstrip(os.path.sep) + os.path.sep)
+        base = sp(os.path.realpath(base_folder)) + os.path.sep
+        subfolder = sp(os.path.realpath(sub_folder)) + os.path.sep
+        return os.path.commonprefix([subfolder, base]) == base
+
+    return False
 
 
 # From SABNZBD
@@ -341,3 +347,36 @@ def removePyc(folder, only_excess = True, show_logs = True):
                     os.rmdir(full_path)
                 except:
                     log.error('Couldn\'t remove empty directory %s: %s', (full_path, traceback.format_exc()))
+
+
+def getFreeSpace(directories):
+
+    single = not isinstance(directories, (tuple, list))
+    if single:
+        directories = [directories]
+
+    free_space = {}
+    for folder in directories:
+
+        size = None
+        if os.path.isdir(folder):
+            if os.name == 'nt':
+                _, total, free = ctypes.c_ulonglong(), ctypes.c_ulonglong(), \
+                                   ctypes.c_ulonglong()
+                if sys.version_info >= (3,) or isinstance(folder, unicode):
+                    fun = ctypes.windll.kernel32.GetDiskFreeSpaceExW #@UndefinedVariable
+                else:
+                    fun = ctypes.windll.kernel32.GetDiskFreeSpaceExA #@UndefinedVariable
+                ret = fun(folder, ctypes.byref(_), ctypes.byref(total), ctypes.byref(free))
+                if ret == 0:
+                    raise ctypes.WinError()
+                return [total.value, free.value]
+            else:
+                s = os.statvfs(folder)
+                size = [s.f_blocks * s.f_frsize / (1024 * 1024), (s.f_bavail * s.f_frsize) / (1024 * 1024)]
+
+        if single: return size
+
+        free_space[folder] = size
+
+    return free_space

couchpotato/core/media/_base/media/main.py

@@ -1,3 +1,6 @@
+from datetime import timedelta
+from operator import itemgetter
+import time
 import traceback
 from string import ascii_lowercase
 
@@ -164,8 +167,15 @@ class MediaPlugin(MediaBase):
         status = list(status if isinstance(status, (list, tuple)) else [status])
 
         for s in status:
-            for ms in db.get_many('media_status', s, with_doc = with_doc):
+            for ms in db.get_many('media_status', s):
-                yield ms['doc'] if with_doc else ms
+                if with_doc:
+                    try:
+                        doc = db.get('id', ms['_id'])
+                        yield doc
+                    except RecordNotFound:
+                        log.debug('Record not found, skipping: %s', ms['_id'])
+                else:
+                    yield ms
 
     def withIdentifiers(self, identifiers, with_doc = False):
 
@@ -282,8 +292,8 @@ class MediaPlugin(MediaBase):
             release_status = splitString(kwargs.get('release_status')),
             status_or = kwargs.get('status_or') is not None,
             limit_offset = kwargs.get('limit_offset'),
-            with_tags = kwargs.get('with_tags'),
+            with_tags = splitString(kwargs.get('with_tags')),
-            starts_with = splitString(kwargs.get('starts_with')),
+            starts_with = kwargs.get('starts_with'),
             search = kwargs.get('search')
         )
 
@@ -401,11 +411,11 @@ class MediaPlugin(MediaBase):
                                 total_deleted += 1
                             new_media_status = 'done'
                         elif delete_from == 'manage':
-                            if release.get('status') == 'done':
+                            if release.get('status') == 'done' or media.get('status') == 'done':
                                 db.delete(release)
                                 total_deleted += 1
 
-                    if (total_releases == total_deleted and media['status'] != 'active') or (not new_media_status and delete_from == 'late'):
+                    if (total_releases == total_deleted and media['status'] != 'active') or (total_releases == 0 and not new_media_status) or (not new_media_status and delete_from == 'late'):
                         db.delete(media)
                         deleted = True
                     elif new_media_status:
@@ -452,28 +462,35 @@ class MediaPlugin(MediaBase):
             if not m['profile_id']:
                 m['status'] = 'done'
             else:
-                move_to_wanted = True
+                m['status'] = 'active'
 
                 try:
                     profile = db.get('id', m['profile_id'])
                     media_releases = fireEvent('release.for_media', m['_id'], single = True)
+                    done_releases = [release for release in media_releases if release.get('status') == 'done']
 
-                    for q_identifier in profile['qualities']:
+                    if done_releases:
-                        index = profile['qualities'].index(q_identifier)
+                        # Only look at latest added release
+                        release = sorted(done_releases, key = itemgetter('last_edit'), reverse = True)[0]
 
-                        for release in media_releases:
+                        # Check if we are finished with the media
-                            if q_identifier == release['quality'] and (release.get('status') == 'done' and profile['finish'][index]):
+                        if fireEvent('quality.isfinish', {'identifier': release['quality'], 'is_3d': release.get('is_3d', False)}, profile, timedelta(seconds = time.time() - release['last_edit']).days, single = True):
-                                move_to_wanted = False
+                            m['status'] = 'done'
+                    elif previous_status == 'done':
+                        m['status'] = 'done'
 
-                    m['status'] = 'active' if move_to_wanted else 'done'
                 except RecordNotFound:
-                    log.debug('Failed restatus: %s', traceback.format_exc())
+                    log.debug('Failed restatus, keeping previous: %s', traceback.format_exc())
+                    m['status'] = previous_status
 
             # Only update when status has changed
             if previous_status != m['status']:
                 db.update(m)
 
-            return True
+                # Tag media as recent
+                self.tag(media_id, 'recent')
+
+            return m['status']
         except:
             log.error('Failed restatus: %s', traceback.format_exc())
 

couchpotato/core/media/_base/providers/torrent/torrentshack.py

@@ -48,9 +48,9 @@ class Base(TorrentProvider):
                         'name': six.text_type(link.span.string).translate({ord(six.u('\xad')): None}),
                         'url': self.urls['download'] % url['href'],
                         'detail_url': self.urls['download'] % link['href'],
-                        'size': self.parseSize(result.find_all('td')[4].string),
+                        'size': self.parseSize(result.find_all('td')[5].string),
-                        'seeders': tryInt(result.find_all('td')[6].string),
+                        'seeders': tryInt(result.find_all('td')[7].string),
-                        'leechers': tryInt(result.find_all('td')[7].string),
+                        'leechers': tryInt(result.find_all('td')[8].string),
                     })
 
             except:

couchpotato/core/media/movie/_base/main.py

@@ -236,7 +236,7 @@ class MovieBase(MovieTypeBase):
 
                     db.update(m)
 
-                    fireEvent('media.restatus', m['_id'])
+                    fireEvent('media.restatus', m['_id'], single = True)
 
                     m = db.get('id', media_id)
 

couchpotato/core/media/movie/providers/info/themoviedb.py

@@ -154,7 +154,7 @@ class TheMovieDb(MovieProvider):
 
             # Add alternative names
             if movie_data['original_title'] and movie_data['original_title'] not in movie_data['titles']:
-                movie_data['titles'].insert(0, movie_data['original_title'])
+                movie_data['titles'].append(movie_data['original_title'])
 
             if extended:
                 for alt in movie.alternate_titles:

couchpotato/core/media/movie/providers/trailer/hdtrailers.py

@@ -21,6 +21,7 @@ class HDTrailers(TrailerProvider):
         'backup': 'http://www.hd-trailers.net/blog/',
     }
     providers = ['apple.ico', 'yahoo.ico', 'moviefone.ico', 'myspace.ico', 'favicon.ico']
+    only_tables_tags = SoupStrainer('table')
 
     def search(self, group):
 
@@ -67,8 +68,7 @@ class HDTrailers(TrailerProvider):
             return results
 
         try:
-            tables = SoupStrainer('div')
+            html = BeautifulSoup(data, 'html.parser', parse_only = self.only_tables_tags)
-            html = BeautifulSoup(data, parse_only = tables)
             result_table = html.find_all('h2', text = re.compile(movie_name))
 
             for h2 in result_table:
@@ -90,8 +90,7 @@ class HDTrailers(TrailerProvider):
 
         results = {'480p':[], '720p':[], '1080p':[]}
         try:
-            tables = SoupStrainer('table')
+            html = BeautifulSoup(data, 'html.parser', parse_only = self.only_tables_tags)
-            html = BeautifulSoup(data, parse_only = tables)
             result_table = html.find('table', attrs = {'class':'bottomTable'})
 
             for tr in result_table.find_all('tr'):

couchpotato/core/media/movie/searcher.py

@@ -120,8 +120,19 @@ class MovieSearcher(SearcherBase, MovieTypeBase):
 
         if not movie['profile_id'] or (movie['status'] == 'done' and not manual):
             log.debug('Movie doesn\'t have a profile or already done, assuming in manage tab.')
+            fireEvent('media.restatus', movie['_id'], single = True)
             return
 
+        default_title = getTitle(movie)
+        if not default_title:
+            log.error('No proper info found for movie, removing it from library to stop it from causing more issues.')
+            fireEvent('media.delete', movie['_id'], single = True)
+            return
+
+        # Update media status and check if it is still not done (due to the stop searching after feature
+        if fireEvent('media.restatus', movie['_id'], single = True) == 'done':
+            log.debug('No better quality found, marking movie %s as done.', default_title)
+
         pre_releases = fireEvent('quality.pre_releases', single = True)
         release_dates = fireEvent('movie.update_release_dates', movie['_id'], merge = True)
 
@@ -133,12 +144,6 @@ class MovieSearcher(SearcherBase, MovieTypeBase):
         ignore_eta = manual
         total_result_count = 0
 
-        default_title = getTitle(movie)
-        if not default_title:
-            log.error('No proper info found for movie, removing it from library to cause it from having more issues.')
-            fireEvent('media.delete', movie['_id'], single = True)
-            return
-
         fireEvent('notify.frontend', type = 'movie.searcher.started', data = {'_id': movie['_id']}, message = 'Searching for "%s"' % default_title)
 
         # Ignore eta once every 7 days
@@ -154,8 +159,7 @@ class MovieSearcher(SearcherBase, MovieTypeBase):
         profile = db.get('id', movie['profile_id'])
         ret = False
 
-        index = 0
+        for index, q_identifier in enumerate(profile.get('qualities', [])):
-        for q_identifier in profile.get('qualities'):
             quality_custom = {
                 'index': index,
                 'quality': q_identifier,
@@ -164,8 +168,6 @@ class MovieSearcher(SearcherBase, MovieTypeBase):
                 '3d': profile['3d'][index] if profile.get('3d') else False
             }
 
-            index += 1
-
             could_not_be_released = not self.couldBeReleased(q_identifier in pre_releases, release_dates, movie['info']['year'])
             if not alway_search and could_not_be_released:
                 too_early_to_search.append(q_identifier)
@@ -189,7 +191,7 @@ class MovieSearcher(SearcherBase, MovieTypeBase):
             # Don't search for quality lower then already available.
             if has_better_quality > 0:
                 log.info('Better quality (%s) already available or snatched for %s', (q_identifier, default_title))
-                fireEvent('media.restatus', movie['_id'])
+                fireEvent('media.restatus', movie['_id'], single = True)
                 break
 
             quality = fireEvent('quality.single', identifier = q_identifier, single = True)

couchpotato/core/media/movie/suggestion/main.py

@@ -1,4 +1,3 @@
-from couchpotato import get_db
 from couchpotato.api import addApiView
 from couchpotato.core.event import fireEvent
 from couchpotato.core.helpers.variable import splitString, removeDuplicate, getIdentifier

couchpotato/core/notifications/xbmc.py

@@ -8,7 +8,7 @@ from couchpotato.core.helpers.variable import splitString, getTitle
 from couchpotato.core.logger import CPLog
 from couchpotato.core.notifications.base import Notification
 import requests
-from requests.packages.urllib3.exceptions import MaxRetryError
+from requests.packages.urllib3.exceptions import MaxRetryError, ConnectionError
 
 
 log = CPLog(__name__)
@@ -172,7 +172,7 @@ class XBMC(Notification):
                 # manually fake expected response array
                 return [{'result': 'Error'}]
 
-        except (MaxRetryError, requests.exceptions.Timeout):
+        except (MaxRetryError, requests.exceptions.Timeout, ConnectionError):
             log.info2('Couldn\'t send request to XBMC, assuming it\'s turned off')
             return [{'result': 'Error'}]
         except:

couchpotato/core/plugins/file.py

@@ -5,7 +5,7 @@ from couchpotato import get_db
 from couchpotato.api import addApiView
 from couchpotato.core.event import addEvent, fireEvent
 from couchpotato.core.helpers.encoding import toUnicode
-from couchpotato.core.helpers.variable import md5, getExt
+from couchpotato.core.helpers.variable import md5, getExt, isSubFolder
 from couchpotato.core.logger import CPLog
 from couchpotato.core.plugins.base import Plugin
 from couchpotato.environment import Env
@@ -32,6 +32,8 @@ class FileManager(Plugin):
 
         fireEvent('schedule.interval', 'file.cleanup', self.cleanup, hours = 24)
 
+        addEvent('app.test', self.doSubfolderTest)
+
     def cleanup(self):
 
         # Wait a bit after starting before cleanup
@@ -76,3 +78,33 @@ class FileManager(Plugin):
 
         self.createFile(dest, filedata, binary = True)
         return dest
+
+    def doSubfolderTest(self):
+
+        tests = {
+            ('/test/subfolder', '/test/sub'): False,
+            ('/test/sub/folder', '/test/sub'): True,
+            ('/test/sub/folder', '/test/sub2'): False,
+            ('/sub/fold', '/test/sub/fold'): False,
+            ('/sub/fold', '/test/sub/folder'): False,
+            ('/opt/couchpotato', '/var/opt/couchpotato'): False,
+            ('/var/opt', '/var/opt/couchpotato'): False,
+            ('/CapItaLs/Are/OK', '/CapItaLs/Are/OK'): True,
+            ('/CapItaLs/Are/OK', '/CapItaLs/Are/OK2'): False,
+            ('/capitals/are/not/OK', '/capitals/are/NOT'): False,
+            ('\\\\Mounted\\Volume\\Test', '\\\\Mounted\\Volume'): True,
+            ('C:\\\\test\\path', 'C:\\\\test2'): False
+        }
+
+        failed = 0
+        for x in tests:
+            if isSubFolder(x[0], x[1]) is not tests[x]:
+                log.error('Failed subfolder test %s %s', x)
+                failed += 1
+
+        if failed > 0:
+            log.error('Subfolder test failed %s tests', failed)
+        else:
+            log.info('Subfolder test succeeded')
+
+        return failed == 0

couchpotato/core/plugins/manage.py

@@ -1,13 +1,12 @@
-import ctypes
 import os
-import sys
 import time
 import traceback
 
+from couchpotato import get_db
 from couchpotato.api import addApiView
 from couchpotato.core.event import fireEvent, addEvent, fireEventAsync
 from couchpotato.core.helpers.encoding import sp
-from couchpotato.core.helpers.variable import splitString, getTitle, tryInt, getIdentifier
+from couchpotato.core.helpers.variable import splitString, getTitle, tryInt, getIdentifier, getFreeSpace
 from couchpotato.core.logger import CPLog
 from couchpotato.core.plugins.base import Plugin
 from couchpotato.environment import Env
@@ -179,6 +178,10 @@ class Manage(Plugin):
                     if self.shuttingDown():
                         break
 
+                if not self.shuttingDown():
+                    db = get_db()
+                    db.reindex()
+
             Env.prop(last_update_key, time.time())
         except:
             log.error('Failed updating library: %s', (traceback.format_exc()))
@@ -268,31 +271,7 @@ class Manage(Plugin):
                         fireEvent('release.add', group = group)
 
     def getDiskSpace(self):
-
+        return getFreeSpace(self.directories())
-        free_space = {}
-        for folder in self.directories():
-
-            size = None
-            if os.path.isdir(folder):
-                if os.name == 'nt':
-                    _, total, free = ctypes.c_ulonglong(), ctypes.c_ulonglong(), \
-                                       ctypes.c_ulonglong()
-                    if sys.version_info >= (3,) or isinstance(folder, unicode):
-                        fun = ctypes.windll.kernel32.GetDiskFreeSpaceExW #@UndefinedVariable
-                    else:
-                        fun = ctypes.windll.kernel32.GetDiskFreeSpaceExA #@UndefinedVariable
-                    ret = fun(folder, ctypes.byref(_), ctypes.byref(total), ctypes.byref(free))
-                    if ret == 0:
-                        raise ctypes.WinError()
-                    used = total.value - free.value
-                    return [total.value, used, free.value]
-                else:
-                    s = os.statvfs(folder)
-                    size = [s.f_blocks * s.f_frsize / (1024 * 1024), (s.f_bavail * s.f_frsize) / (1024 * 1024)]
-
-            free_space[folder] = size
-
-        return free_space
 
 
 config = [{

couchpotato/core/plugins/profile/main.py

@@ -88,6 +88,7 @@ class ProfilePlugin(Plugin):
                 'core': kwargs.get('core', False),
                 'qualities': [],
                 'wait_for': [],
+                'stop_after': [],
                 'finish': [],
                 '3d': []
             }
@@ -97,6 +98,7 @@ class ProfilePlugin(Plugin):
             for type in kwargs.get('types', []):
                 profile['qualities'].append(type.get('quality'))
                 profile['wait_for'].append(tryInt(kwargs.get('wait_for', 0)))
+                profile['stop_after'].append(tryInt(kwargs.get('stop_after', 0)))
                 profile['finish'].append((tryInt(type.get('finish')) == 1) if order > 0 else True)
                 profile['3d'].append(tryInt(type.get('3d')))
                 order += 1
@@ -217,6 +219,7 @@ class ProfilePlugin(Plugin):
                     'qualities': profile.get('qualities'),
                     'finish': [],
                     'wait_for': [],
+                    'stop_after': [],
                     '3d': []
                 }
 
@@ -224,6 +227,7 @@ class ProfilePlugin(Plugin):
                 for q in profile.get('qualities'):
                     pro['finish'].append(True)
                     pro['wait_for'].append(0)
+                    pro['stop_after'].append(0)
                     pro['3d'].append(threed.pop() if threed else False)
 
                 db.insert(pro)

couchpotato/core/plugins/profile/static/profile.css

@@ -43,9 +43,8 @@
 	}
 
 	.profile .wait_for {
-		position: absolute;
+		padding-top: 0;
-		right: 60px;
+		padding-bottom: 20px;
-		top: 0;
 	}
 
 	.profile .wait_for input {

couchpotato/core/plugins/profile/static/profile.js

@@ -37,20 +37,28 @@ var Profile = new Class({
 					'placeholder': 'Profile name'
 				})
 			),
-			new Element('div.wait_for.ctrlHolder').adopt(
-				new Element('span', {'text':'Wait'}),
-				new Element('input.inlay.xsmall', {
-					'type':'text',
-					'value': data.wait_for && data.wait_for.length > 0 ? data.wait_for[0] : 0
-				}),
-				new Element('span', {'text':'day(s) for a better quality.'})
-			),
 			new Element('div.qualities.ctrlHolder').adopt(
 				new Element('label', {'text': 'Search for'}),
 				self.type_container = new Element('ol.types'),
 				new Element('div.formHint', {
 					'html': "Search these qualities (2 minimum), from top to bottom. Use the checkbox, to stop searching after it found this quality."
 				})
+			),
+			new Element('div.wait_for.ctrlHolder').adopt(
+				// "Wait the entered number of days for a checked quality, before downloading a lower quality release."
+				new Element('span', {'text':'Wait'}),
+				new Element('input.inlay.wait_for_input.xsmall', {
+					'type':'text',
+					'value': data.wait_for && data.wait_for.length > 0 ? data.wait_for[0] : 0
+				}),
+				new Element('span', {'text':'day(s) for a better quality '}),
+				new Element('span.advanced', {'text':'and keep searching'}),
+				// "After a checked quality is found and downloaded, continue searching for even better quality releases for the entered number of days."
+				new Element('input.inlay.xsmall.stop_after_input.advanced', {
+					'type':'text',
+					'value': data.stop_after && data.stop_after.length > 0 ? data.stop_after[0] : 0
+				}),
+				new Element('span.advanced', {'text':'day(s) for a better (checked) quality.'})
 			)
 		);
 
@@ -116,7 +124,8 @@ var Profile = new Class({
 		var data = {
 			'id' : self.data._id,
 			'label' : self.el.getElement('.quality_label input').get('value'),
-			'wait_for' : self.el.getElement('.wait_for input').get('value'),
+			'wait_for' : self.el.getElement('.wait_for_input').get('value'),
+			'stop_after' : self.el.getElement('.stop_after_input').get('value'),
 			'types': []
 		};
 

couchpotato/core/plugins/quality/main.py

@@ -25,14 +25,14 @@ class QualityPlugin(Plugin):
         {'identifier': 'bd50', 'hd': True, 'allow_3d': True, 'size': (20000, 60000), 'label': 'BR-Disk', 'alternative': ['bd25', ('br', 'disk')], 'allow': ['1080p'], 'ext':['iso', 'img'], 'tags': ['bdmv', 'certificate', ('complete', 'bluray'), 'avc', 'mvc']},
         {'identifier': '1080p', 'hd': True, 'allow_3d': True, 'size': (4000, 20000), 'label': '1080p', 'width': 1920, 'height': 1080, 'alternative': [], 'allow': [], 'ext':['mkv', 'm2ts', 'ts'], 'tags': ['m2ts', 'x264', 'h264']},
         {'identifier': '720p', 'hd': True, 'allow_3d': True, 'size': (3000, 10000), 'label': '720p', 'width': 1280, 'height': 720, 'alternative': [], 'allow': [], 'ext':['mkv', 'ts'], 'tags': ['x264', 'h264']},
-        {'identifier': 'brrip', 'hd': True, 'allow_3d': True, 'size': (700, 7000), 'label': 'BR-Rip', 'alternative': ['bdrip', ('br', 'rip')], 'allow': ['720p', '1080p'], 'ext':[], 'tags': ['hdtv', 'hdrip', 'webdl', ('web', 'dl')]},
+        {'identifier': 'brrip', 'hd': True, 'allow_3d': True, 'size': (700, 7000), 'label': 'BR-Rip', 'alternative': ['bdrip', ('br', 'rip')], 'allow': ['720p', '1080p'], 'ext':['mp4', 'avi'], 'tags': ['hdtv', 'hdrip', 'webdl', ('web', 'dl')]},
         {'identifier': 'dvdr', 'size': (3000, 10000), 'label': 'DVD-R', 'alternative': ['br2dvd', ('dvd', 'r')], 'allow': [], 'ext':['iso', 'img', 'vob'], 'tags': ['pal', 'ntsc', 'video_ts', 'audio_ts', ('dvd', 'r'), 'dvd9']},
-        {'identifier': 'dvdrip', 'size': (600, 2400), 'label': 'DVD-Rip', 'width': 720, 'alternative': [('dvd', 'rip')], 'allow': [], 'ext':[], 'tags': [('dvd', 'rip'), ('dvd', 'xvid'), ('dvd', 'divx')]},
+        {'identifier': 'dvdrip', 'size': (600, 2400), 'label': 'DVD-Rip', 'width': 720, 'alternative': [('dvd', 'rip')], 'allow': [], 'ext':['avi'], 'tags': [('dvd', 'rip'), ('dvd', 'xvid'), ('dvd', 'divx')]},
         {'identifier': 'scr', 'size': (600, 1600), 'label': 'Screener', 'alternative': ['screener', 'dvdscr', 'ppvrip', 'dvdscreener', 'hdscr'], 'allow': ['dvdr', 'dvdrip', '720p', '1080p'], 'ext':[], 'tags': ['webrip', ('web', 'rip')]},
         {'identifier': 'r5', 'size': (600, 1000), 'label': 'R5', 'alternative': ['r6'], 'allow': ['dvdr'], 'ext':[]},
         {'identifier': 'tc', 'size': (600, 1000), 'label': 'TeleCine', 'alternative': ['telecine'], 'allow': [], 'ext':[]},
         {'identifier': 'ts', 'size': (600, 1000), 'label': 'TeleSync', 'alternative': ['telesync', 'hdts'], 'allow': [], 'ext':[]},
-        {'identifier': 'cam', 'size': (600, 1000), 'label': 'Cam', 'alternative': ['camrip', 'hdcam'], 'allow': [], 'ext':[]}
+        {'identifier': 'cam', 'size': (600, 1000), 'label': 'Cam', 'alternative': ['camrip', 'hdcam'], 'allow': ['720p'], 'ext':[]}
     ]
     pre_releases = ['cam', 'ts', 'tc', 'r5', 'scr']
     threed_tags = {
@@ -379,26 +379,31 @@ class QualityPlugin(Plugin):
                 if score.get(q.get('identifier')):
                     score[q.get('identifier')]['score'] -= 1
 
-    def isFinish(self, quality, profile):
+    def isFinish(self, quality, profile, release_age = 0):
         if not isinstance(profile, dict) or not profile.get('qualities'):
-            return False
+            # No profile so anything (scanned) is good enough
+            return True
 
         try:
-            quality_order = [i for i, identifier in enumerate(profile['qualities']) if identifier == quality['identifier'] and bool(profile['3d'][i] if profile.get('3d') else 0) == bool(quality.get('is_3d', 0))][0]
+            index = [i for i, identifier in enumerate(profile['qualities']) if identifier == quality['identifier'] and bool(profile['3d'][i] if profile.get('3d') else False) == bool(quality.get('is_3d', False))][0]
-            return profile['finish'][quality_order]
+
+            if index == 0 or (profile['finish'][index] and int(release_age) >= int(profile.get('stop_after', [0])[0])):
+                return True
+
+            return False
         except:
             return False
 
     def isHigher(self, quality, compare_with, profile = None):
         if not isinstance(profile, dict) or not profile.get('qualities'):
-            profile = {'qualities': self.order}
+            profile = fireEvent('profile.default', single = True)
 
         # Try to find quality in profile, if not found: a quality we do not want is lower than anything else
         try:
             quality_order = [i for i, identifier in enumerate(profile['qualities']) if identifier == quality['identifier'] and bool(profile['3d'][i] if profile.get('3d') else 0) == bool(quality.get('is_3d', 0))][0]
         except:
             log.debug('Quality %s not found in profile identifiers %s', (quality['identifier'] + (' 3D' if quality.get('is_3d', 0) else ''), \
-                [identifier + ('3D' if (profile['3d'][i] if profile.get('3d') else 0) else '') for i, identifier in enumerate(profile['qualities'])]))
+                [identifier + (' 3D' if (profile['3d'][i] if profile.get('3d') else 0) else '') for i, identifier in enumerate(profile['qualities'])]))
             return 'lower'
 
         # Try to find compare quality in profile, if not found: anything is higher than a not wanted quality
@@ -446,6 +451,9 @@ class QualityPlugin(Plugin):
             '/movies/BluRay HDDVD H.264 MKV 720p EngSub/QuiQui le fou (criterion collection #123, 1915)/QuiQui le fou (1915) 720p x264 BluRay.mkv': {'size': 5500, 'quality': '720p'},
             'C:\\movies\QuiQui le fou (collection #123, 1915)\QuiQui le fou (1915) 720p x264 BluRay.mkv': {'size': 5500, 'quality': '720p'},
             'C:\\movies\QuiQui le fou (collection #123, 1915)\QuiQui le fou (1915) half-sbs 720p x264 BluRay.mkv': {'size': 5500, 'quality': '720p', 'is_3d': True},
+            'Moviename 2014 720p HDCAM XviD DualAudio': {'size': 4000, 'quality': 'cam'},
+            'Moviename (2014) - 720p CAM x264': {'size': 2250, 'quality': 'cam'},
+            'Movie Name (2014).mp4': {'size': 750, 'quality': 'brrip'},
         }
 
         correct = 0

couchpotato/core/plugins/release/main.py

@@ -3,7 +3,7 @@ import os
 import time
 import traceback
 
-from CodernityDB.database import RecordDeleted
+from CodernityDB.database import RecordDeleted, RecordNotFound
 from couchpotato import md5, get_db
 from couchpotato.api import addApiView
 from couchpotato.core.event import fireEvent, addEvent
@@ -79,6 +79,13 @@ class Release(Plugin):
             try:
                 db.get('id', release.get('key'))
                 media_exist.append(release.get('key'))
+
+                try:
+                    if release['doc'].get('status') == 'ignore':
+                        release['doc']['status'] = 'ignored'
+                        db.update(release['doc'])
+                except:
+                    log.error('Failed fixing mis-status tag: %s', traceback.format_exc())
             except RecordDeleted:
                 db.delete(release['doc'])
                 log.debug('Deleted orphaned release: %s', release['doc'])
@@ -100,9 +107,9 @@ class Release(Plugin):
                 if rel['status'] in ['available']:
                     self.delete(rel['_id'])
 
-                # Set all snatched and downloaded releases to ignored to make sure they are ignored when re-adding the move
+                # Set all snatched and downloaded releases to ignored to make sure they are ignored when re-adding the media
                 elif rel['status'] in ['snatched', 'downloaded']:
-                    self.updateStatus(rel['_id'], status = 'ignore')
+                    self.updateStatus(rel['_id'], status = 'ignored')
 
             fireEvent('media.untag', media.get('_id'), 'recent', single = True)
 
@@ -164,7 +171,7 @@ class Release(Plugin):
             release['files'] = dict((k, [toUnicode(x) for x in v]) for k, v in group['files'].items() if v)
             db.update(release)
 
-            fireEvent('media.restatus', media['_id'])
+            fireEvent('media.restatus', media['_id'], single = True)
 
             return True
         except:
@@ -331,24 +338,14 @@ class Release(Plugin):
 
                 if media['status'] == 'active':
                     profile = db.get('id', media['profile_id'])
-                    finished = False
+                    if fireEvent('quality.isfinish', {'identifier': rls['quality'], 'is_3d': rls.get('is_3d', False)}, profile, single = True):
-                    if rls['quality'] in profile['qualities']:
-                        nr = profile['qualities'].index(rls['quality'])
-                        finished = profile['finish'][nr]
-
-                    if finished:
                         log.info('Renamer disabled, marking media as finished: %s', log_movie)
 
                         # Mark release done
                         self.updateStatus(rls['_id'], status = 'done')
 
                         # Mark media done
-                        mdia = db.get('id', media['_id'])
+                        fireEvent('media.restatus', media['_id'], single = True)
-                        mdia['status'] = 'done'
-                        mdia['last_edit'] = int(time.time())
-                        db.update(mdia)
-
-                        fireEvent('media.tag', media['_id'], 'recent', single = True)
 
                         return True
 
@@ -511,8 +508,15 @@ class Release(Plugin):
         status = list(status if isinstance(status, (list, tuple)) else [status])
 
         for s in status:
-            for ms in db.get_many('release_status', s, with_doc = with_doc):
+            for ms in db.get_many('release_status', s):
-                yield ms['doc'] if with_doc else ms
+                if with_doc:
+                    try:
+                        doc = db.get('id', ms['_id'])
+                        yield doc
+                    except RecordNotFound:
+                        log.debug('Record not found, skipping: %s', ms['_id'])
+                else:
+                    yield ms
 
     def forMedia(self, media_id):
 

couchpotato/core/plugins/renamer.py

@@ -136,7 +136,7 @@ class Renamer(Plugin):
         else:
             for item in no_process:
                 if isSubFolder(item, base_folder):
-                    log.error('To protect your data, the media libraries can\'t be inside of or the same as the "from" folder.')
+                    log.error('To protect your data, the media libraries can\'t be inside of or the same as the "from" folder. "%s" in "%s"', (item, base_folder))
                     return
 
         # Check to see if the no_process folders are inside the provided media_folder
@@ -168,7 +168,7 @@ class Renamer(Plugin):
         if media_folder:
             for item in no_process:
                 if isSubFolder(item, media_folder):
-                    log.error('To protect your data, the media libraries can\'t be inside of or the same as the provided media folder.')
+                    log.error('To protect your data, the media libraries can\'t be inside of or the same as the provided media folder. "%s" in "%s"', (item, media_folder))
                     return
 
         # Make sure a checkSnatched marked all downloads/seeds as such
@@ -446,22 +446,19 @@ class Renamer(Plugin):
                 # Before renaming, remove the lower quality files
                 remove_leftovers = True
 
-                # Mark movie "done" once it's found the quality with the finish check
+                # Get media quality profile
                 profile = None
-                try:
+                if media.get('profile_id'):
-                    if media.get('status') == 'active' and media.get('profile_id'):
+                    try:
                         profile = db.get('id', media['profile_id'])
-                        if fireEvent('quality.isfinish', group['meta_data']['quality'], profile, single = True):
+                    except:
-                            mdia = db.get('id', media['_id'])
+                        # Set profile to None as it does not exist anymore
-                            mdia['status'] = 'done'
+                        mdia = db.get('id', media['_id'])
-                            mdia['last_edit'] = int(time.time())
+                        mdia['profile_id'] = None
-                            db.update(mdia)
+                        db.update(mdia)
-
+                        log.error('Error getting quality profile for %s: %s', (media_title, traceback.format_exc()))
-                            # List movie on dashboard
+                else:
-                            fireEvent('media.tag', media['_id'], 'recent', single = True)
+                    log.debug('Media has no quality profile: %s', media_title)
-
-                except:
-                    log.error('Failed marking movie finished: %s', (traceback.format_exc()))
 
                 # Mark media for dashboard
                 mark_as_recent = False
@@ -474,7 +471,7 @@ class Renamer(Plugin):
 
                         # This is where CP removes older, lesser quality releases or releases that are not wanted anymore
                         is_higher = fireEvent('quality.ishigher', \
-                            group['meta_data']['quality'], {'identifier': release['quality'], 'is_3d': release.get('is_3d', 0)}, profile, single = True)
+                            group['meta_data']['quality'], {'identifier': release['quality'], 'is_3d': release.get('is_3d', False)}, profile, single = True)
 
                         if is_higher == 'higher':
                             log.info('Removing lesser or not wanted quality %s for %s.', (media_title, release.get('quality')))
@@ -499,7 +496,7 @@ class Renamer(Plugin):
                             self.tagRelease(group = group, tag = 'exists')
 
                             # Notify on rename fail
-                            download_message = 'Renaming of %s (%s) cancelled, exists in %s already.' % (media_title, group['meta_data']['quality']['label'], release.get('identifier'))
+                            download_message = 'Renaming of %s (%s) cancelled, exists in %s already.' % (media_title, group['meta_data']['quality']['label'], release.get('quality'))
                             fireEvent('movie.renaming.canceled', message = download_message, data = group)
                             remove_leftovers = False
 
@@ -518,7 +515,7 @@ class Renamer(Plugin):
                                     fireEvent('release.update_status', release['_id'], status = 'seeding', single = True)
                                     mark_as_recent = True
 
-                        elif release.get('identifier') == group['meta_data']['quality']['identifier']:
+                        elif release.get('quality') == group['meta_data']['quality']['identifier']:
                             # Set the release to downloaded
                             fireEvent('release.update_status', release['_id'], status = 'downloaded', single = True)
                             group['release_download'] = release_download

couchpotato/core/plugins/scanner.py

@@ -639,9 +639,9 @@ class Scanner(Plugin):
 
                         # Try with other
                         if len(movie) == 0 and name_year.get('other') and name_year['other'].get('name') and name_year['other'].get('year'):
-                            search_q2 = '%(name)s %(year)s' % name_year
+                            search_q2 = '%(name)s %(year)s' % name_year.get('other')
                             if search_q2 != search_q:
-                                movie = fireEvent('movie.search', q = '%(name)s %(year)s' % name_year.get('other'), merge = True, limit = 1)
+                                movie = fireEvent('movie.search', q = search_q2, merge = True, limit = 1)
 
                         if len(movie) > 0:
                             imdb_id = movie[0].get('imdb')

couchpotato/core/plugins/trailer.py

@@ -32,7 +32,7 @@ class Trailer(Plugin):
             destination = os.path.join(group['destination_dir'], filename)
             if not os.path.isfile(destination):
                 trailer_file = fireEvent('file.download', url = trailer, dest = destination, urlopen_kwargs = {'headers': {'User-Agent': 'Quicktime'}}, single = True)
-                if os.path.getsize(trailer_file) < (1024 * 1024):  # Don't trust small trailers (1MB), try next one
+                if trailer_file and os.path.getsize(trailer_file) < (1024 * 1024):  # Don't trust small trailers (1MB), try next one
                     os.unlink(trailer_file)
                     continue
             else:

couchpotato/core/settings.py

@@ -71,15 +71,7 @@ class Settings(object):
         self.connectEvents()
 
     def databaseSetup(self):
-        from couchpotato import get_db
+        fireEvent('database.setup_index', 'property', PropertyIndex)
-
-        db = get_db()
-
-        try:
-            db.add_index(PropertyIndex(db.path, 'property'))
-        except:
-            self.log.debug('Index for properties already exists')
-            db.edit_index(PropertyIndex(db.path, 'property'))
 
     def parser(self):
         return self.p

couchpotato/runner.py

@@ -17,7 +17,7 @@ from couchpotato import KeyHandler, LoginHandler, LogoutHandler
 from couchpotato.api import NonBlockHandler, ApiHandler
 from couchpotato.core.event import fireEventAsync, fireEvent
 from couchpotato.core.helpers.encoding import sp
-from couchpotato.core.helpers.variable import getDataDir, tryInt
+from couchpotato.core.helpers.variable import getDataDir, tryInt, getFreeSpace
 import requests
 from tornado.httpserver import HTTPServer
 from tornado.web import Application, StaticFileHandler, RedirectHandler
@@ -195,6 +195,15 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En
     log = CPLog(__name__)
     log.debug('Started with options %s', options)
 
+    # Check available space
+    try:
+        total_space, available_space = getFreeSpace(data_dir)
+        if available_space < 100:
+            log.error('Shutting down as CP needs some space to work. You\'ll get corrupted data otherwise. Only %sMB left', available_space)
+            return
+    except:
+        log.error('Failed getting diskspace: %s', traceback.format_exc())
+
     def customwarn(message, category, filename, lineno, file = None, line = None):
         log.warning('%s %s %s line:%s', (category, message, filename, lineno))
     warnings.showwarning = customwarn

couchpotato/static/style/settings.css

@@ -75,6 +75,8 @@
 			color: #edc07f;
 		}
 		.page.show_advanced .advanced { display: block; }
+		.page.show_advanced span.advanced,
+		.page.show_advanced input.advanced { display: inline; }
 
 		.page.settings .tab_content {
 			display: none;
@@ -176,7 +178,7 @@
 			padding: 6px 0 0;
 		}
 
-		.page .xsmall { width: 20px !important; text-align: center; }
+		.page .xsmall { width: 25px !important; text-align: center; }
 
 	.page .enabler {
 		display: block;