From 69819460f393b69e7e190c3c8d52fa87120cf74c Mon Sep 17 00:00:00 2001 From: Ruud Date: Wed, 25 Jun 2014 09:27:23 +0200 Subject: [PATCH 1/4] Add zoink.it for torrent caching --- couchpotato/core/_base/downloader/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/couchpotato/core/_base/downloader/main.py b/couchpotato/core/_base/downloader/main.py index 70e5cc9c..7ef98af2 100644 --- a/couchpotato/core/_base/downloader/main.py +++ b/couchpotato/core/_base/downloader/main.py @@ -25,6 +25,7 @@ class DownloaderBase(Provider): status_support = True torrent_sources = [ + 'https://zoink.it/torrent/%s.torrent', 'http://torrage.com/torrent/%s.torrent', 'https://torcache.net/torrent/%s.torrent', ] From a566b4f4284904acc7592ed72f5ba8a25011fc27 Mon Sep 17 00:00:00 2001 From: Ruud Date: Wed, 25 Jun 2014 16:13:35 +0200 Subject: [PATCH 2/4] Setup property index with database module --- couchpotato/core/settings.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/couchpotato/core/settings.py b/couchpotato/core/settings.py index c6de952b..4315ec18 100644 --- a/couchpotato/core/settings.py +++ b/couchpotato/core/settings.py @@ -71,15 +71,7 @@ class Settings(object): self.connectEvents() def databaseSetup(self): - from couchpotato import get_db - - db = get_db() - - try: - db.add_index(PropertyIndex(db.path, 'property')) - except: - self.log.debug('Index for properties already exists') - db.edit_index(PropertyIndex(db.path, 'property')) + fireEvent('database.setup_index', 'property', PropertyIndex) def parser(self): return self.p From ffc3fc9ec456bf0b908912038d3c22190a60dc13 Mon Sep 17 00:00:00 2001 From: Ruud Date: Wed, 25 Jun 2014 18:16:45 +0200 Subject: [PATCH 3/4] Check for broken indexes and reindex if needed --- couchpotato/core/database.py | 98 +++++++++++++++++++++++++++++------- 1 file changed, 81 insertions(+), 17 deletions(-) diff --git a/couchpotato/core/database.py b/couchpotato/core/database.py index d753d87f..c7051f5c 100644 --- a/couchpotato/core/database.py +++ b/couchpotato/core/database.py @@ -3,10 +3,11 @@ import os import time import traceback +from CodernityDB.index import IndexException, IndexNotFoundException, IndexConflict from couchpotato import CPLog from couchpotato.api import addApiView from couchpotato.core.event import addEvent, fireEvent -from couchpotato.core.helpers.encoding import toUnicode +from couchpotato.core.helpers.encoding import toUnicode, sp from couchpotato.core.helpers.variable import getImdb, tryInt @@ -15,11 +16,13 @@ log = CPLog(__name__) class Database(object): - indexes = [] + indexes = None db = None def __init__(self): + self.indexes = {} + addApiView('database.list_documents', self.listDocuments) addApiView('database.reindex', self.reindex) addApiView('database.compact', self.compact) @@ -45,26 +48,45 @@ class Database(object): def setupIndex(self, index_name, klass): - self.indexes.append(index_name) + self.indexes[index_name] = klass db = self.getDB() # Category index index_instance = klass(db.path, index_name) try: - db.add_index(index_instance) - db.reindex_index(index_name) - except: - previous = db.indexes_names[index_name] - previous_version = previous._version - current_version = klass._version - # Only edit index if versions are different - if previous_version < current_version: - log.debug('Index "%s" already exists, updating and reindexing', index_name) - db.destroy_index(previous) + # Make sure store and bucket don't exist + exists = [] + for x in ['buck', 'stor']: + full_path = os.path.join(db.path, '%s_%s' % (index_name, x)) + if os.path.exists(full_path): + exists.append(full_path) + + if index_name not in db.indexes_names: + + # Remove existing buckets if index isn't there + for x in exists: + os.unlink(x) + + # Add index (will restore buckets) db.add_index(index_instance) db.reindex_index(index_name) + else: + # Previous info + previous = db.indexes_names[index_name] + previous_version = previous._version + current_version = klass._version + + # Only edit index if versions are different + if previous_version < current_version: + log.debug('Index "%s" already exists, updating and reindexing', index_name) + db.destroy_index(previous) + db.add_index(index_instance) + db.reindex_index(index_name) + + except: + log.error('Failed adding index %s: %s', (index_name, traceback.format_exc())) def deleteDocument(self, **kwargs): @@ -138,21 +160,62 @@ class Database(object): 'success': success } - def compact(self, **kwargs): + def compact(self, try_repair = True, **kwargs): + + success = False + db = self.getDB() + + # Removing left over compact files + db_path = sp(db.path) + for f in os.listdir(sp(db.path)): + for x in ['_compact_buck', '_compact_stor']: + if f[-len(x):] == x: + os.unlink(os.path.join(db_path, f)) - success = True try: start = time.time() - db = self.getDB() size = float(db.get_db_details().get('size', 0)) log.debug('Compacting database, current size: %sMB', round(size/1048576, 2)) db.compact() new_size = float(db.get_db_details().get('size', 0)) log.debug('Done compacting database in %ss, new size: %sMB, saved: %sMB', (round(time.time()-start, 2), round(new_size/1048576, 2), round((size-new_size)/1048576, 2))) + success = True + except (IndexException, AttributeError): + if try_repair: + log.error('Something wrong with indexes, trying repair') + + # Remove all indexes + old_indexes = self.indexes.keys() + for index_name in old_indexes: + try: + db.destroy_index(index_name) + except IndexNotFoundException: + pass + except: + log.error('Failed removing old index %s', index_name) + + # Add them again + for index_name in self.indexes: + klass = self.indexes[index_name] + + # Category index + index_instance = klass(db.path, index_name) + try: + db.add_index(index_instance) + db.reindex_index(index_name) + except IndexConflict: + pass + except: + log.error('Failed adding index %s', index_name) + raise + + self.compact(try_repair = False) + else: + log.error('Failed compact: %s', traceback.format_exc()) + except: log.error('Failed compact: %s', traceback.format_exc()) - success = False return { 'success': success @@ -166,6 +229,7 @@ class Database(object): size = db.get_db_details().get('size') prop_name = 'last_db_compact' last_check = int(Env.prop(prop_name, default = 0)) + if size > 26214400 and last_check < time.time()-604800: # 25MB / 7 days self.compact() Env.prop(prop_name, value = int(time.time())) From 9ec4c2837e7e2588078e756f5ce43464f8b2cb9f Mon Sep 17 00:00:00 2001 From: Ruud Date: Wed, 25 Jun 2014 18:20:42 +0200 Subject: [PATCH 4/4] Don't put original title first --- couchpotato/core/media/movie/providers/info/themoviedb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/media/movie/providers/info/themoviedb.py b/couchpotato/core/media/movie/providers/info/themoviedb.py index 4a397edd..ac1daecd 100644 --- a/couchpotato/core/media/movie/providers/info/themoviedb.py +++ b/couchpotato/core/media/movie/providers/info/themoviedb.py @@ -154,7 +154,7 @@ class TheMovieDb(MovieProvider): # Add alternative names if movie_data['original_title'] and movie_data['original_title'] not in movie_data['titles']: - movie_data['titles'].insert(0, movie_data['original_title']) + movie_data['titles'].append(movie_data['original_title']) if extended: for alt in movie.alternate_titles: