database manage init

couchpotato/core/database.py

@@ -0,0 +1,109 @@
+import json
+import time
+import traceback
+from couchpotato import CPLog
+from couchpotato.api import addApiView
+from couchpotato.core.event import addEvent
+
+log = CPLog(__name__)
+
+
+class Database(object):
+
+    indexes = []
+    db = None
+
+    def __init__(self):
+
+        addApiView('database.list_documents', self.listDocuments)
+        addApiView('database.document.update', self.updateDocument)
+        addApiView('database.document.delete', self.deleteDocument)
+
+        addEvent('database.setup_index', self.setupIndex)
+
+        def test():
+            time.sleep(1)
+            self.listDocuments()
+
+        addEvent('app.load', test)
+
+
+    def getDB(self):
+
+        if not self.db:
+            from couchpotato import get_db
+            self.db = get_db()
+
+        return self.db
+
+    def setupIndex(self, index_name, klass):
+
+        self.indexes.append(index_name)
+
+        db = self.getDB()
+
+        # Category index
+        try:
+            db.add_index(klass(db.path, index_name))
+            db.reindex_index(index_name)
+        except:
+            previous_version = db.indexes_names[index_name]._version
+            current_version = klass._version
+
+            # Only edit index if versions are different
+            if previous_version < current_version:
+                log.debug('Index "%s" already exists, updating and reindexing', index_name)
+                db.edit_index(klass(db.path, index_name), reindex = True)
+
+    def deleteDocument(self, **kwargs):
+
+        db = self.getDB()
+
+        try:
+            document = db.get(id)
+            db.delete(document)
+
+            return {
+                'success': True
+            }
+        except:
+            return {
+                'success': False,
+                'error': traceback.format_exc()
+            }
+
+    def updateDocument(self, **kwargs):
+
+        db = self.getDB()
+
+        try:
+
+            document = json.loads(kwargs.get('_request').get_argument('document'))
+            d = db.update(document)
+            document.update(d)
+
+            return {
+                'success': True,
+                'document': document
+            }
+        except:
+            return {
+                'success': False,
+                'error': traceback.format_exc()
+            }
+
+    def listDocuments(self, **kwargs):
+        db = self.getDB()
+
+        results = {
+            'unknown': []
+        }
+
+        for document in db.all('id'):
+            key = document.get('_t', 'unknown')
+            if not results.get(key):
+                results[key] = []
+            results[key].append(document)
+
+
+        return results

couchpotato/core/media/__init__.py

@@ -10,15 +10,6 @@ class MediaBase(Plugin):
 
     _type = None
 
-    default_dict = {
-        'profile': {'types': {'quality': {}}},
-        'releases': {'status': {}, 'quality': {}, 'files': {}, 'info': {}},
-        'library': {'titles': {}, 'files': {}},
-        'files': {},
-        'status': {},
-        'category': {},
-    }
-
     def initType(self):
         addEvent('media.types', self.getType)
 

couchpotato/core/media/_base/media/index.py

@@ -7,6 +7,7 @@ from couchpotato.core.helpers.encoding import toUnicode, simplifyString
 
 
 class MediaIMDBIndex(HashIndex):
+    _version = 1
 
     def __init__(self, *args, **kwargs):
         kwargs['key_format'] = 'I'
@@ -19,8 +20,8 @@ class MediaIMDBIndex(HashIndex):
         if data.get('_t') == 'media' and data.get('identifier'):
             return int(data['identifier'].strip('t')), None
 
-    def run_to_dict(self, db, media_id, dict = None):
-        if not dict: dict = {}
+    def run_to_dict(self, db, media_id, dict_dept = None):
+        if not dict_dept: dict_dept = {}
 
         return db.get('id', media_id)
 
@@ -34,6 +35,7 @@ class MediaIMDBIndex(HashIndex):
 
 
 class MediaStatusIndex(TreeBasedIndex):
+    _version = 1
 
     def __init__(self, *args, **kwargs):
         kwargs['key_format'] = '32s'
@@ -48,6 +50,7 @@ class MediaStatusIndex(TreeBasedIndex):
 
 
 class MediaTypeIndex(TreeBasedIndex):
+    _version = 1
 
     def __init__(self, *args, **kwargs):
         kwargs['key_format'] = '32s'
@@ -62,6 +65,7 @@ class MediaTypeIndex(TreeBasedIndex):
 
 
 class TitleSearchIndex(MultiTreeBasedIndex):
+    _version = 1
 
     custom_header = """from CodernityDB.tree_index import MultiTreeBasedIndex
 from itertools import izip
@@ -93,6 +97,7 @@ from couchpotato.core.helpers.encoding import simplifyString"""
 
 
 class TitleIndex(TreeBasedIndex):
+    _version = 1
 
     custom_header = """from CodernityDB.tree_index import TreeBasedIndex
 from string import ascii_letters
@@ -125,6 +130,7 @@ from couchpotato.core.helpers.encoding import toUnicode, simplifyString"""
 
 
 class StartsWithIndex(TreeBasedIndex):
+    _version = 1
 
     custom_header = """from CodernityDB.tree_index import TreeBasedIndex
 from string import ascii_letters

couchpotato/core/media/_base/media/main.py

@@ -14,6 +14,15 @@ log = CPLog(__name__)
 
 class MediaPlugin(MediaBase):
 
+    _database = {
+        'media': MediaIMDBIndex,
+        'media_search_title': MediaStatusIndex,
+        'media_status': MediaTypeIndex,
+        'media_by_type': TitleSearchIndex,
+        'media_title': TitleIndex,
+        'media_startswith': StartsWithIndex,
+    }
+
     def __init__(self):
 
         addApiView('media.refresh', self.refresh, docs = {
@@ -57,8 +66,6 @@ class MediaPlugin(MediaBase):
 
         addApiView('media.available_chars', self.charView)
 
-        addEvent('database.setup', self.databaseSetup)
-
         addEvent('app.load', self.addSingleRefreshView, priority = 100)
         addEvent('app.load', self.addSingleListView, priority = 100)
         addEvent('app.load', self.addSingleCharView, priority = 100)
@@ -69,51 +76,6 @@ class MediaPlugin(MediaBase):
         addEvent('media.delete', self.delete)
         addEvent('media.restatus', self.restatus)
 
-    def databaseSetup(self):
-
-        db = get_db()
-
-        # IMDB index
-        try:
-            db.add_index(MediaIMDBIndex(db.path, 'media'))
-        except:
-            log.debug('Index already exists')
-            db.edit_index(MediaIMDBIndex(db.path, 'media'))
-
-        # Title index
-        try:
-            db.add_index(TitleSearchIndex(db.path, 'media_search_title'))
-        except:
-            log.debug('Index already exists')
-            db.edit_index(TitleSearchIndex(db.path, 'media_search_title'))
-
-        # Status index
-        try:
-            db.add_index(MediaStatusIndex(db.path, 'media_status'))
-        except:
-            log.debug('Index already exists')
-            db.edit_index(MediaStatusIndex(db.path, 'media_status'))
-
-        # Type index
-        try:
-            db.add_index(MediaTypeIndex(db.path, 'media_by_type'))
-        except:
-            log.debug('Index already exists')
-            db.edit_index(MediaTypeIndex(db.path, 'media_by_type'))
-
-        # Title index
-        try: db.add_index(TitleIndex(db.path, 'media_title'))
-        except:
-            log.debug('Index already exists')
-            db.edit_index(TitleIndex(db.path, 'media_title'))
-
-        # Startswith index
-        try: db.add_index(StartsWithIndex(db.path, 'media_startswith'))
-        except:
-            log.debug('Index already exists')
-            db.edit_index(StartsWithIndex(db.path, 'media_startswith'))
-
-
     def refresh(self, id = '', **kwargs):
         handlers = []
         ids = splitString(id)

couchpotato/core/notifications/core/index.py

@@ -3,6 +3,7 @@ from CodernityDB.tree_index import TreeBasedIndex
 
 
 class NotificationIndex(TreeBasedIndex):
+    _version = 1
 
     custom_header = """from CodernityDB.tree_index import TreeBasedIndex
 import time"""
@@ -23,6 +24,7 @@ import time"""
 
 
 class NotificationUnreadIndex(TreeBasedIndex):
+    _version = 1
 
     custom_header = """from CodernityDB.tree_index import TreeBasedIndex
 import time"""

couchpotato/core/notifications/core/main.py

@@ -18,6 +18,11 @@ log = CPLog(__name__)
 
 class CoreNotifier(Notification):
 
+    _database = {
+        'notification': NotificationIndex,
+        'notification_unread': NotificationUnreadIndex
+    }
+
     m_lock = None
 
     listen_to = [
@@ -60,28 +65,10 @@ class CoreNotifier(Notification):
         addEvent('app.load', self.clean)
         addEvent('app.load', self.checkMessages)
 
-        addEvent('database.setup', self.databaseSetup)
-
         self.messages = []
         self.listeners = []
         self.m_lock = threading.Lock()
 
-    def databaseSetup(self):
-
-        db = get_db()
-
-        try:
-            db.add_index(NotificationIndex(db.path, 'notification'))
-        except:
-            log.debug('Index already exists')
-            db.edit_index(NotificationIndex(db.path, 'notification'))
-
-        try:
-            db.add_index(NotificationUnreadIndex(db.path, 'notification_unread'))
-        except:
-            log.debug('Index already exists')
-            db.edit_index(NotificationUnreadIndex(db.path, 'notification_unread'))
-
     def clean(self):
         try:
             db = get_db()

couchpotato/core/plugins/base.py

@@ -1,3 +1,4 @@
+from couchpotato import get_db
 from couchpotato.core.event import fireEvent, addEvent
 from couchpotato.core.helpers.encoding import ss, toSafeString, \
     toUnicode, sp
@@ -24,6 +25,7 @@ log = CPLog(__name__)
 class Plugin(object):
 
     _class_name = None
+    _database = None
     plugin_path = None
 
     enabled_option = 'enabled'
@@ -53,6 +55,22 @@ class Plugin(object):
         if self.auto_register_static:
             self.registerStatic(inspect.getfile(self.__class__))
 
+        # Setup database
+        if self._database:
+            addEvent('database.setup', self.databaseSetup)
+
+    def databaseSetup(self):
+
+        db = get_db()
+
+        for index_name in self._database:
+            klass = self._database[index_name]
+
+            fireEvent('database.setup_index', index_name, klass)
+
+    def afterDatabaseSetup(self):
+        print self._database_indexes
+
     def conf(self, attr, value = None, default = None, section = None):
         class_name = self.getName().lower().split(':')[0].lower()
         return Env.setting(attr, section = section if section else class_name, value = value, default = default)

couchpotato/core/plugins/category/index.py

@@ -2,6 +2,7 @@ from CodernityDB.tree_index import TreeBasedIndex
 
 
 class CategoryIndex(TreeBasedIndex):
+    _version = 1
 
     def __init__(self, *args, **kwargs):
         kwargs['key_format'] = 'i'
@@ -16,6 +17,7 @@ class CategoryIndex(TreeBasedIndex):
 
 
 class CategoryMediaIndex(TreeBasedIndex):
+    _version = 1
 
     def __init__(self, *args, **kwargs):
         kwargs['key_format'] = '32s'

couchpotato/core/plugins/category/main.py

@@ -12,6 +12,11 @@ log = CPLog(__name__)
 
 class CategoryPlugin(Plugin):
 
+    _database = {
+        'category': CategoryIndex,
+        'category_media': CategoryMediaIndex,
+    }
+
     def __init__(self):
         addApiView('category.save', self.save)
         addApiView('category.save_order', self.saveOrder)
@@ -25,25 +30,6 @@ class CategoryPlugin(Plugin):
         })
 
         addEvent('category.all', self.all)
-        addEvent('database.setup', self.databaseSetup)
-
-    def databaseSetup(self):
-
-        db = get_db()
-
-        # Category index
-        try:
-            db.add_index(CategoryIndex(db.path, 'category'))
-        except:
-            log.debug('Index already exists')
-            db.edit_index(CategoryIndex(db.path, 'category'))
-
-        # Category media_id index
-        try:
-            db.add_index(CategoryMediaIndex(db.path, 'category_media'))
-        except:
-            log.debug('Index already exists')
-            db.edit_index(CategoryMediaIndex(db.path, 'category_media'))
 
     def allView(self, **kwargs):
 

couchpotato/core/plugins/file/main.py

@@ -47,9 +47,9 @@ class FileManager(Plugin):
                 for x in file_dict.keys():
                     files.extend(file_dict[x])
 
-            for file in scandir.scandir(cache_dir):
-                if os.path.splitext(file.name)[1] in ['.png', '.jpg', '.jpeg']:
-                    file_path = os.path.join(cache_dir, file.name)
+            for f in scandir.scandir(cache_dir):
+                if os.path.splitext(f.name)[1] in ['.png', '.jpg', '.jpeg']:
+                    file_path = os.path.join(cache_dir, f.name)
                     if toUnicode(file_path) not in files:
                         os.remove(file_path)
         except:

couchpotato/core/plugins/profile/index.py

@@ -2,6 +2,7 @@ from CodernityDB.tree_index import TreeBasedIndex
 
 
 class ProfileIndex(TreeBasedIndex):
+    _version = 1
 
     def __init__(self, *args, **kwargs):
         kwargs['key_format'] = 'i'

couchpotato/core/plugins/profile/main.py

@@ -12,7 +12,9 @@ log = CPLog(__name__)
 
 class ProfilePlugin(Plugin):
 
-    to_dict = {'types': {}}
+    _database = {
+        'profile': ProfileIndex
+    }
 
     def __init__(self):
         addEvent('profile.all', self.all)
@@ -29,21 +31,9 @@ class ProfilePlugin(Plugin):
 }"""}
         })
 
-        addEvent('database.setup', self.databaseSetup)
-
         addEvent('app.initialize', self.fill, priority = 90)
         addEvent('app.load', self.forceDefaults)
 
-    def databaseSetup(self):
-
-        db = get_db()
-
-        try:
-            db.add_index(ProfileIndex(db.path, 'profile'))
-        except:
-            log.debug('Index already exists')
-            db.edit_index(ProfileIndex(db.path, 'profile'))
-
     def forceDefaults(self):
 
         # Get all active movies without profile

couchpotato/core/plugins/quality/index.py

@@ -3,6 +3,7 @@ from hashlib import md5
 
 
 class QualityIndex(HashIndex):
+    _version = 1
 
     def __init__(self, *args, **kwargs):
         kwargs['key_format'] = '32s'

couchpotato/core/plugins/quality/main.py

@@ -14,6 +14,10 @@ log = CPLog(__name__)
 
 class QualityPlugin(Plugin):
 
+    _database = {
+        'quality': QualityIndex
+    }
+
     qualities = [
         {'identifier': 'bd50', 'hd': True, 'size': (15000, 60000), 'label': 'BR-Disk', 'alternative': ['bd25'], 'allow': ['1080p'], 'ext':[], 'tags': ['bdmv', 'certificate', ('complete', 'bluray')]},
         {'identifier': '1080p', 'hd': True, 'size': (4000, 20000), 'label': '1080p', 'width': 1920, 'height': 1080, 'alternative': [], 'allow': [], 'ext':['mkv', 'm2ts'], 'tags': ['m2ts', 'x264', 'h264']},
@@ -49,7 +53,6 @@ class QualityPlugin(Plugin):
         })
 
         addEvent('app.initialize', self.fill, priority = 10)
-        addEvent('database.setup', self.databaseSetup)
 
         addEvent('app.test', self.doTest)
 
@@ -63,17 +66,6 @@ class QualityPlugin(Plugin):
     def getOrder(self):
         return self.order
 
-    def databaseSetup(self):
-
-        db = get_db()
-
-        # Quality index
-        try:
-            db.add_index(QualityIndex(db.path, 'quality'))
-        except:
-            log.debug('Index already exists')
-            db.edit_index(QualityIndex(db.path, 'quality'))
-
     def preReleases(self):
         return self.pre_releases
 

couchpotato/core/plugins/release/index.py

@@ -4,6 +4,7 @@ from CodernityDB.tree_index import TreeBasedIndex
 
 
 class ReleaseIndex(TreeBasedIndex):
+    _version = 1
 
     def __init__(self, *args, **kwargs):
         kwargs['key_format'] = '32s'
@@ -30,6 +31,7 @@ class ReleaseIndex(TreeBasedIndex):
 
 
 class ReleaseStatusIndex(TreeBasedIndex):
+    _version = 1
 
     def __init__(self, *args, **kwargs):
         kwargs['key_format'] = '32s'
@@ -44,6 +46,7 @@ class ReleaseStatusIndex(TreeBasedIndex):
 
 
 class ReleaseIDIndex(HashIndex):
+    _version = 1
 
     def __init__(self, *args, **kwargs):
         kwargs['key_format'] = '32s'
@@ -58,6 +61,7 @@ class ReleaseIDIndex(HashIndex):
 
 
 class ReleaseDownloadIndex(HashIndex):
+    _version = 1
 
     def __init__(self, *args, **kwargs):
         kwargs['key_format'] = '32s'

couchpotato/core/plugins/release/main.py

@@ -17,6 +17,13 @@ log = CPLog(__name__)
 
 class Release(Plugin):
 
+    _database = {
+        'release': ReleaseIndex,
+        'release_status': ReleaseStatusIndex,
+        'release_identifier': ReleaseIDIndex,
+        'release_download': ReleaseDownloadIndex
+    }
+
     def __init__(self):
         addApiView('release.manual_download', self.manualDownload, docs = {
             'desc': 'Send a release manually to the downloaders',
@@ -45,44 +52,10 @@ class Release(Plugin):
         addEvent('release.clean', self.clean)
         addEvent('release.update_status', self.updateStatus)
 
-        addEvent('database.setup', self.databaseSetup)
-
         # Clean releases that didn't have activity in the last week
         addEvent('app.load', self.cleanDone)
         fireEvent('schedule.interval', 'movie.clean_releases', self.cleanDone, hours = 4)
 
-    def databaseSetup(self):
-
-        db = get_db()
-
-        # Release media_id index
-        try:
-            db.add_index(ReleaseIndex(db.path, 'release'))
-        except:
-            log.debug('Index already exists')
-            db.edit_index(ReleaseIndex(db.path, 'release'))
-
-        # Release status index
-        try:
-            db.add_index(ReleaseStatusIndex(db.path, 'release_status'))
-        except:
-            log.debug('Index already exists')
-            db.edit_index(ReleaseStatusIndex(db.path, 'release_status'))
-
-        # Release identifier index
-        try:
-            db.add_index(ReleaseIDIndex(db.path, 'release_identifier'))
-        except:
-            log.debug('Index already exists')
-            db.edit_index(ReleaseIDIndex(db.path, 'release_identifier'))
-
-        # Release identifier index
-        try:
-            db.add_index(ReleaseDownloadIndex(db.path, 'release_download'))
-        except:
-            log.debug('Index already exists')
-            db.edit_index(ReleaseDownloadIndex(db.path, 'release_download'))
-
     def cleanDone(self):
         log.debug('Removing releases from dashboard')
 

couchpotato/core/plugins/userscript/template.js

@@ -141,4 +141,4 @@ if(document.location.href.indexOf(host) == -1)
 else
     setVersion();
 
-}
+}

couchpotato/core/providers/torrent/torrentpotato/main.py

@@ -75,7 +75,7 @@ class TorrentPotato(TorrentProvider):
         pass_keys = splitString(self.conf('pass_key'), clean = False)
         extra_score = splitString(self.conf('extra_score'), clean = False)
 
-        list = []
+        host_list = []
         for nr in range(len(hosts)):
 
             try: key = pass_keys[nr]
@@ -93,7 +93,7 @@ class TorrentPotato(TorrentProvider):
             try: seed_time = seed_times[nr]
             except: seed_time = ''
 
-            list.append({
+            host_list.append({
                 'use': uses[nr],
                 'host': host,
                 'name': name,
@@ -103,7 +103,7 @@ class TorrentPotato(TorrentProvider):
                 'extra_score': tryInt(extra_score[nr]) if len(extra_score) > nr else 0
             })
 
-        return list
+        return host_list
 
     def belongsTo(self, url, provider = None, host = None):
 

couchpotato/core/settings.py

@@ -1,11 +1,12 @@
 from __future__ import with_statement
 import traceback
+from CodernityDB.hash_index import HashIndex
 from couchpotato.api import addApiView
 from couchpotato.core.event import addEvent, fireEvent
 from couchpotato.core.helpers.encoding import toUnicode
 from couchpotato.core.helpers.variable import mergeDicts, tryInt, tryFloat
-from couchpotato.core.settings.index import PropertyIndex
 import ConfigParser
+from hashlib import md5
 
 
 class Settings(object):
@@ -73,7 +74,7 @@ class Settings(object):
         try:
             db.add_index(PropertyIndex(db.path, 'property'))
         except:
-            self.log.debug('Index already exists')
+            self.log.debug('Index for properties already exists')
             db.edit_index(PropertyIndex(db.path, 'property'))
 
     def parser(self):
@@ -250,3 +251,17 @@ class Settings(object):
                 'identifier': identifier,
                 'value': toUnicode(value),
             })
+
+class PropertyIndex(HashIndex):
+    _version = 1
+
+    def __init__(self, *args, **kwargs):
+        kwargs['key_format'] = '32s'
+        super(PropertyIndex, self).__init__(*args, **kwargs)
+
+    def make_key(self, key):
+        return md5(key).hexdigest()
+
+    def make_key_value(self, data):
+        if data.get('_t') == 'property':
+            return md5(data['identifier']).hexdigest(), None

couchpotato/core/settings/index.py

@@ -1,16 +0,0 @@
-from CodernityDB.hash_index import UniqueHashIndex, HashIndex
-from hashlib import md5
-
-
-class PropertyIndex(HashIndex):
-
-    def __init__(self, *args, **kwargs):
-        kwargs['key_format'] = '32s'
-        super(PropertyIndex, self).__init__(*args, **kwargs)
-
-    def make_key(self, key):
-        return md5(key).hexdigest()
-
-    def make_key_value(self, data):
-        if data.get('_t') == 'property':
-            return md5(data['identifier']).hexdigest(), None