Try NoSQL

couchpotato/__init__.py

@@ -49,7 +49,11 @@ def addView(route, func, static = False):
 
 
 def get_session():
-    return Env.getSession()
+    return None
+
+
+def get_db():
+    return Env.get('db')
 
 
 # Web view

couchpotato/core/_base/_core/main.py

@@ -42,11 +42,11 @@ class Core(Plugin):
 
         addEvent('app.shutdown', self.shutdown)
         addEvent('app.restart', self.restart)
-        addEvent('app.load', self.launchBrowser, priority = 1)
+        addEvent('app.load2', self.launchBrowser, priority = 1)
         addEvent('app.base_url', self.createBaseUrl)
         addEvent('app.api_url', self.createApiUrl)
         addEvent('app.version', self.version)
-        addEvent('app.load', self.checkDataDir)
+        addEvent('app.load2', self.checkDataDir)
 
         addEvent('setting.save.core.password', self.md5Password)
         addEvent('setting.save.core.api_key', self.checkApikey)

couchpotato/core/_base/clientscript/main.py

@@ -74,7 +74,7 @@ class ClientScript(Plugin):
         addEvent('clientscript.get_scripts', self.getScripts)
 
         if not Env.get('dev'):
-            addEvent('app.load', self.minify)
+            addEvent('app.load2', self.minify)
 
         self.addCore()
 

couchpotato/core/_base/desktop/main.py

@@ -25,7 +25,7 @@ if Env.get('desktop'):
 
             # Events to desktop
             addEvent('app.after_shutdown', desktop.afterShutdown)
-            addEvent('app.load', desktop.onAppLoad, priority = 110)
+            addEvent('app.load2', desktop.onAppLoad, priority = 110)
 
         def onClose(self, event):
             return fireEvent('app.shutdown', single = True)

couchpotato/core/_base/updater/main.py

@@ -33,8 +33,8 @@ class Updater(Plugin):
         else:
             self.updater = SourceUpdater()
 
-        addEvent('app.load', self.logVersion, priority = 10000)
+        addEvent('app.load2', self.logVersion, priority = 10000)
-        addEvent('app.load', self.setCrons)
+        addEvent('app.load2', self.setCrons)
         addEvent('updater.info', self.info)
 
         addApiView('updater.info', self.info, docs = {

couchpotato/core/downloaders/rtorrent/main.py

@@ -23,7 +23,7 @@ class rTorrent(Downloader):
     def __init__(self):
         super(rTorrent, self).__init__()
 
-        addEvent('app.load', self.migrate)
+        addEvent('app.load2', self.migrate)
 
     def migrate(self):
 

couchpotato/core/media/__init__.py

@@ -1,9 +1,11 @@
 import traceback
-from couchpotato import get_session
+from couchpotato import get_session, get_db, CPLog
 from couchpotato.core.event import addEvent, fireEventAsync, fireEvent
 from couchpotato.core.plugins.base import Plugin
 from couchpotato.core.settings.model import Media
 
+log = CPLog(__name__)
+
 
 class MediaBase(Plugin):
 
@@ -24,20 +26,19 @@ class MediaBase(Plugin):
     def getType(self):
         return self._type
 
-    def createOnComplete(self, id):
+    def createOnComplete(self, media_id):
 
         def onComplete():
             try:
-                db = get_session()
+                db = get_db()
-                media = db.query(Media).filter_by(id = id).first()
+                media = db.get('id', media_id)
-                media_dict = media.to_dict(self.default_dict)
+                event_name = '%s.searcher.single' % media.get('type')
-                event_name = '%s.searcher.single' % media.type
 
-                fireEvent(event_name, media_dict, on_complete = self.createNotifyFront(id))
+                fireEvent(event_name, media, on_complete = self.createNotifyFront(media_id))
             except:
                 log.error('Failed creating onComplete: %s', traceback.format_exc())
             finally:
-                db.close()
+                pass  #db.close()
 
         return onComplete
 
@@ -45,15 +46,14 @@ class MediaBase(Plugin):
 
         def notifyFront():
             try:
-                db = get_session()
+                db = get_db()
-                media = db.query(Media).filter_by(id = media_id).first()
+                media = db.get('id', media_id)
-                media_dict = media.to_dict(self.default_dict)
+                event_name = '%s.update' % media.get('type')
-                event_name = '%s.update' % media.type
 
-                fireEvent('notify.frontend', type = event_name, data = media_dict)
+                fireEvent('notify.frontend', type = event_name, data = media)
             except:
                 log.error('Failed creating onComplete: %s', traceback.format_exc())
             finally:
-                db.close()
+                pass  #db.close()
 
         return notifyFront

couchpotato/core/media/_base/media/index.py

@@ -0,0 +1,60 @@
+from CodernityDB.hash_index import HashIndex
+from CodernityDB.tree_index import MultiTreeBasedIndex
+
+
+class MediaIMDBIndex(HashIndex):
+
+    def __init__(self, *args, **kwargs):
+        kwargs['key_format'] = 'I'
+        super(MediaIMDBIndex, self).__init__(*args, **kwargs)
+
+    def make_key(self, key):
+        return int(key.strip('t'))
+
+    def make_key_value(self, data):
+        if data.get('type') == 'media' and data.get('identifier'):
+            return int(data['identifier'].strip('t')), None
+
+
+class MediaStatusIndex(HashIndex):
+
+    def __init__(self, *args, **kwargs):
+        kwargs['key_format'] = 's'
+        super(MediaStatusIndex, self).__init__(*args, **kwargs)
+
+    def make_key(self, key):
+        return int(key.strip('t'))
+
+    def make_key_value(self, data):
+        if data.get('type') == 'media' and data.get('identifier'):
+            return int(data['identifier'].strip('t')), None
+
+
+class TitleIndex(MultiTreeBasedIndex):
+
+    custom_header = """from CodernityDB.tree_index import MultiTreeBasedIndex
+from itertools import izip"""
+
+    def __init__(self, *args, **kwargs):
+        kwargs['key_format'] = '32s'
+        super(TitleIndex, self).__init__(*args, **kwargs)
+        self.__l = kwargs.get('w_len', 2)
+
+    def make_key_value(self, data):
+
+        if data.get('type') == 'title' and len(data.get('title', '')) > 0:
+
+            out = set()
+            title = data.get('title').lower()
+            l = self.__l
+            max_l = len(title)
+            for x in xrange(l - 1, max_l):
+                m = (title, )
+                for y in xrange(0, x):
+                    m += (title[y + 1:],)
+                out.update(set(''.join(x).rjust(32, '_').lower() for x in izip(*m)))  #ignore import error
+
+            return out, {'media_id': data.get('media_id')}
+
+    def make_key(self, key):
+        return key.rjust(32, '_').lower()

couchpotato/core/media/_base/media/main.py

@@ -1,11 +1,12 @@
 import traceback
-from couchpotato import get_session, tryInt
+from couchpotato import get_session, tryInt, get_db
 from couchpotato.api import addApiView
 from couchpotato.core.event import fireEvent, fireEventAsync, addEvent
 from couchpotato.core.helpers.encoding import toUnicode
 from couchpotato.core.helpers.variable import mergeDicts, splitString, getImdb, getTitle
 from couchpotato.core.logger import CPLog
 from couchpotato.core.media import MediaBase
+from .index import MediaIMDBIndex, TitleIndex, MediaStatusIndex
 from couchpotato.core.settings.model import Library, LibraryTitle, Release, \
     Media
 from sqlalchemy.orm import joinedload_all
@@ -60,16 +61,40 @@ class MediaPlugin(MediaBase):
 
         addApiView('media.available_chars', self.charView)
 
-        addEvent('app.load', self.addSingleRefreshView)
+        addEvent('database.setup', self.databaseSetup)
-        addEvent('app.load', self.addSingleListView)
+
-        addEvent('app.load', self.addSingleCharView)
+        addEvent('app.load2', self.addSingleRefreshView)
-        addEvent('app.load', self.addSingleDeleteView)
+        addEvent('app.load2', self.addSingleListView)
+        addEvent('app.load2', self.addSingleCharView)
+        addEvent('app.load2', self.addSingleDeleteView)
 
         addEvent('media.get', self.get)
         addEvent('media.list', self.list)
         addEvent('media.delete', self.delete)
         addEvent('media.restatus', self.restatus)
 
+    def databaseSetup(self):
+
+        db = get_db()
+
+        try:
+            db.add_index(MediaIMDBIndex(db.path, 'media'))
+        except:
+            log.debug('Index already exists')
+            db.update_index(MediaIMDBIndex(db.path, 'media'))
+
+        try:
+            db.add_index(TitleIndex(db.path, 'media_title'))
+        except:
+            log.debug('Index already exists')
+            db.update_index(TitleIndex(db.path, 'media_title'))
+
+        try:
+            db.add_index(MediaStatusIndex(db.path, 'media_status'))
+        except:
+            log.debug('Index already exists')
+            db.update_index(MediaStatusIndex(db.path, 'media_status'))
+
     def refresh(self, id = '', **kwargs):
         handlers = []
         ids = splitString(id)
@@ -87,24 +112,23 @@ class MediaPlugin(MediaBase):
             'success': True,
         }
 
-    def createRefreshHandler(self, id):
+    def createRefreshHandler(self, media_id):
-        db = get_session()
 
-        media = db.query(Media).filter_by(id = id).first()
+        try:
+            media = get_db().get('id', media_id)
 
-        if media:
+            default_title = getTitle(media_id)
-
+            event = 'library.update.%s' % media.get('type')
-            default_title = getTitle(media.library)
-            identifier = media.library.identifier
-            event = 'library.update.%s' % media.type
 
             def handler():
-                fireEvent(event, identifier = identifier, default_title = default_title, on_complete = self.createOnComplete(id))
+                fireEvent(event, identifier = media.get('identifier'), default_title = default_title, on_complete = self.createOnComplete(media_id))
 
-        db.close()
+            if handler:
+                return handler
+
+        except:
+            log.error('Refresh handler for non existing media: %s', traceback.format_exc())
 
-        if handler:
-            return handler
 
     def addSingleRefreshView(self):
 
@@ -113,20 +137,19 @@ class MediaPlugin(MediaBase):
 
     def get(self, media_id):
 
-        db = get_session()
+        db = get_db()
 
         imdb_id = getImdb(str(media_id))
 
         if imdb_id:
-            m = db.query(Media).filter(Media.library.has(identifier = imdb_id)).first()
+            m = db.get('media', imdb_id, with_doc = True)['doc']
         else:
-            m = db.query(Media).filter_by(id = media_id).first()
+            m = db.get('id', media_id)
 
         results = None
         if m:
-            results = m.to_dict(self.default_dict)
+            results = db.run('media', 'to_dict', m, self.default_dict)
 
-        db.close()
         return results
 
     def getView(self, id = None, **kwargs):
@@ -262,7 +285,7 @@ class MediaPlugin(MediaBase):
                 'releases_count': releases_count.get(media_id),
             }))
 
-        db.close()
+        pass  #db.close()
         return total_count, movies
 
     def listView(self, **kwargs):
@@ -356,7 +379,7 @@ class MediaPlugin(MediaBase):
             if len(chars) == 25:
                 break
 
-        db.close()
+        pass  #db.close()
         return ''.join(sorted(chars))
 
     def charView(self, **kwargs):
@@ -428,7 +451,7 @@ class MediaPlugin(MediaBase):
             log.error('Failed deleting media: %s', traceback.format_exc())
             db.rollback()
         finally:
-            db.close()
+            pass  #db.close()
 
         return True
 
@@ -481,5 +504,5 @@ class MediaPlugin(MediaBase):
             log.error('Failed restatus: %s', traceback.format_exc())
             db.rollback()
         finally:
-            db.close()
+            pass  #db.close()
 

couchpotato/core/media/_base/search/main.py

@@ -25,7 +25,7 @@ class Search(Plugin):
 }"""}
         })
 
-        addEvent('app.load', self.addSingleSearches)
+        addEvent('app.load2', self.addSingleSearches)
 
     def search(self, q = '', types = None, **kwargs):
 

couchpotato/core/media/_base/searcher/base.py

@@ -28,7 +28,7 @@ class SearcherBase(Plugin):
             fireEvent('schedule.cron', '%s.searcher.all' % _type, self.searchAll,
                       day = self.conf('cron_day'), hour = self.conf('cron_hour'), minute = self.conf('cron_minute'))
 
-        addEvent('app.load', setCrons)
+        addEvent('app.load2', setCrons)
         addEvent('setting.save.%s_searcher.cron_day.after' % _type, setCrons)
         addEvent('setting.save.%s_searcher.cron_hour.after' % _type, setCrons)
         addEvent('setting.save.%s_searcher.cron_minute.after' % _type, setCrons)

couchpotato/core/media/movie/_base/main.py

@@ -1,5 +1,5 @@
 import traceback
-from couchpotato import get_session
+from couchpotato import get_session, get_db
 from couchpotato.api import addApiView
 from couchpotato.core.event import fireEvent, fireEventAsync, addEvent
 from couchpotato.core.helpers.encoding import toUnicode
@@ -62,78 +62,78 @@ class MovieBase(MovieTypeBase):
             except:
                 pass
 
-        library = fireEvent('library.add.movie', single = True, attrs = params, update_after = update_library)
+        # library = fireEvent('library.add.movie', single = True, attrs = params, update_after = update_library)
-
+        info = fireEvent('movie.info', merge = True, extended = False, identifier = params.get('identifier'))
-        # Status
-        status_active, snatched_status, ignored_status, done_status, downloaded_status = \
-            fireEvent('status.get', ['active', 'snatched', 'ignored', 'done', 'downloaded'], single = True)
 
         default_profile = fireEvent('profile.default', single = True)
         cat_id = params.get('category_id')
 
         try:
-            db = get_session()
+            db = get_db()
-            m = db.query(Media).filter_by(library_id = library.get('id')).first()
+
+            new = False
+            try:
+                m = db.get('movie', params.get('identifier'), with_doc = True)['doc']
+            except:
+                new = True
+                m = db.insert({
+                    'type': 'movie',
+                    'identifier': params.get('identifier'),
+                    'status': status_id if status_id else 'active',
+                    'profile_id': params.get('profile_id', default_profile.get('id')),
+                    'category_id': tryInt(cat_id) if cat_id is not None and tryInt(cat_id) > 0 else None,
+                })
+
             added = True
             do_search = False
             search_after = search_after and self.conf('search_on_add', section = 'moviesearcher')
-            if not m:
+            if new:
-                m = Media(
-                    library_id = library.get('id'),
-                    profile_id = params.get('profile_id', default_profile.get('id')),
-                    status_id = status_id if status_id else status_active.get('id'),
-                    category_id = tryInt(cat_id) if cat_id is not None and tryInt(cat_id) > 0 else None,
-                )
-                db.add(m)
-                db.commit()
-
                 onComplete = None
                 if search_after:
-                    onComplete = self.createOnComplete(m.id)
+                    onComplete = self.createOnComplete(m['_id'])
 
-                fireEventAsync('library.update.movie', params.get('identifier'), default_title = params.get('title', ''), on_complete = onComplete)
+                # fireEventAsync('library.update.movie', params.get('identifier'), default_title = params.get('title', ''), on_complete = onComplete)
                 search_after = False
             elif force_readd:
 
                 # Clean snatched history
-                for release in m.releases:
+                for release in db.run('release', 'for_media', m['_id']):
-                    if release.status_id in [downloaded_status.get('id'), snatched_status.get('id'), done_status.get('id')]:
+                    if release.get('status') in ['downloaded', 'snatched', 'done']:
                         if params.get('ignore_previous', False):
-                            release.status_id = ignored_status.get('id')
+                            release['status'] = 'ignored'
+                            db.update(release)
                         else:
-                            fireEvent('release.delete', release.id, single = True)
+                            fireEvent('release.delete', release['_id'], single = True)
 
-                m.profile_id = params.get('profile_id', default_profile.get('id'))
+                m['profile_id'] = params.get('profile_id', default_profile.get('id'))
-                m.category_id = tryInt(cat_id) if cat_id is not None and tryInt(cat_id) > 0 else (m.category_id or None)
+                m['category_id'] = tryInt(cat_id) if cat_id is not None and tryInt(cat_id) > 0 else (m['category_id'] or None)
             else:
                 log.debug('Movie already exists, not updating: %s', params)
                 added = False
 
             if force_readd:
-                m.status_id = status_id if status_id else status_active.get('id')
+                m['status'] = status_id if status_id else 'active'
-                m.last_edit = int(time.time())
+                m['last_edit'] = int(time.time())
                 do_search = True
 
-            db.commit()
+            db.update(m)
 
             # Remove releases
-            available_status = fireEvent('status.get', 'available', single = True)
+            for rel in db.run('release', 'for_media', m['_id']):
-            for rel in m.releases:
+                if rel['status'] is 'available':
-                if rel.status_id is available_status.get('id'):
                     db.delete(rel)
-                    db.commit()
 
-            movie_dict = m.to_dict(self.default_dict)
+            movie_dict = db.run('movie', 'to_dict', m['_id'])
 
             if do_search and search_after:
-                onComplete = self.createOnComplete(m.id)
+                onComplete = self.createOnComplete(m['_id'])
                 onComplete()
 
             if added:
                 if params.get('title'):
                     message = 'Successfully added "%s" to your wanted list.' % params.get('title', '')
                 else:
-                    title = getTitle(m.library)
+                    title = getTitle(m)
                     if title:
                         message = 'Successfully added "%s" to your wanted list.' % title
                     else:
@@ -142,10 +142,7 @@ class MovieBase(MovieTypeBase):
 
             return movie_dict
         except:
-            log.error('Failed deleting media: %s', traceback.format_exc())
+            log.error('Failed adding media: %s', traceback.format_exc())
-            db.rollback()
-        finally:
-            db.close()
 
     def addView(self, **kwargs):
         add_dict = self.add(params = kwargs)
@@ -158,49 +155,44 @@ class MovieBase(MovieTypeBase):
     def edit(self, id = '', **kwargs):
 
         try:
-            db = get_session()
+            db = get_db()
-
-            available_status = fireEvent('status.get', 'available', single = True)
 
             ids = splitString(id)
             for media_id in ids:
 
-                m = db.query(Media).filter_by(id = media_id).first()
+                try:
-                if not m:
+                    m = db.get('media', media_id)
-                    continue
+                    m['profile_id'] = kwargs.get('profile_id')
 
-                m.profile_id = kwargs.get('profile_id')
+                    cat_id = kwargs.get('category_id')
+                    if cat_id is not None:
+                        m['category_id'] = tryInt(cat_id) if tryInt(cat_id) > 0 else None
 
-                cat_id = kwargs.get('category_id')
+                    # Remove releases
-                if cat_id is not None:
+                    for rel in db.run('release', 'for_media', m['_id']):
-                    m.category_id = tryInt(cat_id) if tryInt(cat_id) > 0 else None
+                        if rel['status'] is 'available':
+                            db.delete(rel)
 
-                # Remove releases
+                    # Default title
-                for rel in m.releases:
+                    if kwargs.get('default_title'):
-                    if rel.status_id is available_status.get('id'):
+                        for title in m['titles']:
-                        db.delete(rel)
+                            title.default = toUnicode(kwargs.get('default_title', '')).lower() == toUnicode(title.title).lower()
-                        db.commit()
 
-                # Default title
+                    db.update(m)
-                if kwargs.get('default_title'):
-                    for title in m.library.titles:
-                        title.default = toUnicode(kwargs.get('default_title', '')).lower() == toUnicode(title.title).lower()
 
-                db.commit()
+                    fireEvent('media.restatus', m['_id'])
 
-                fireEvent('media.restatus', m.id)
+                    movie_dict = db.run('media', 'to_dict', m['_id'])
+                    fireEventAsync('movie.searcher.single', movie_dict, on_complete = self.createNotifyFront(media_id))
 
-                movie_dict = m.to_dict(self.default_dict)
+                except:
-                fireEventAsync('movie.searcher.single', movie_dict, on_complete = self.createNotifyFront(media_id))
+                    log.error('Can\'t edit non-existing media')
 
             return {
                 'success': True,
             }
         except:
-            log.error('Failed deleting media: %s', traceback.format_exc())
+            log.error('Failed editing media: %s', traceback.format_exc())
-            db.rollback()
-        finally:
-            db.close()
 
         return {
             'success': False,

couchpotato/core/media/movie/library/movie/main.py

@@ -62,7 +62,7 @@ class MovieLibraryPlugin(LibraryBase):
             log.error('Failed adding media: %s', traceback.format_exc())
             db.rollback()
         finally:
-            db.close()
+            pass  #db.close()
 
         return {}
 
@@ -130,31 +130,24 @@ class MovieLibraryPlugin(LibraryBase):
 
             # Files
             images = info.get('images', [])
+            library['files'] = []
             for image_type in ['poster']:
                 for image in images.get(image_type, []):
                     if not isinstance(image, (str, unicode)):
                         continue
 
                     file_path = fireEvent('file.download', url = image, single = True)
-                    if file_path:
-                        file_obj = fireEvent('file.add', path = file_path, type_tuple = ('image', image_type), single = True)
-                        try:
-                            file_obj = db.query(File).filter_by(id = file_obj.get('id')).one()
-                            library.files.append(file_obj)
-                            db.commit()
 
-                            break
+                    # TODO: save in movie doc
-                        except:
+                    library['files'].append({
-                            log.debug('Failed to attach to library: %s', traceback.format_exc())
+                        'type': 'image_%s' % image_type,
-                            db.rollback()
+                        'path': file_path
+                    })
 
             library_dict = library.to_dict(self.default_dict)
             return library_dict
         except:
             log.error('Failed update media: %s', traceback.format_exc())
-            db.rollback()
-        finally:
-            db.close()
 
         return {}
 
@@ -180,7 +173,7 @@ class MovieLibraryPlugin(LibraryBase):
             log.error('Failed updating release dates: %s', traceback.format_exc())
             db.rollback()
         finally:
-            db.close()
+            pass  #db.close()
 
         return {}
 

couchpotato/core/media/movie/searcher/main.py

@@ -1,4 +1,4 @@
-from couchpotato import get_session
+from couchpotato import get_session, get_db
 from couchpotato.api import addApiView
 from couchpotato.core.event import addEvent, fireEvent, fireEventAsync
 from couchpotato.core.helpers.encoding import simplifyString
@@ -51,7 +51,7 @@ class MovieSearcher(SearcherBase, MovieTypeBase):
         })
 
         if self.conf('run_on_launch'):
-            addEvent('app.load', self.searchAll)
+            addEvent('app.load2', self.searchAll)
 
     def searchAllView(self, **kwargs):
 
@@ -71,27 +71,16 @@ class MovieSearcher(SearcherBase, MovieTypeBase):
         self.in_progress = True
         fireEvent('notify.frontend', type = 'movie.searcher.started', data = True, message = 'Full search started')
 
-        db = get_session()
+        db = get_db()
 
-        movies_raw = db.query(Media).filter(
+        movies = db.get_many('movie_status', 'active')
-            Media.status.has(identifier = 'active')
-        ).all()
 
-        random.shuffle(movies_raw)
+        #TODO: random.shuffle(movies_raw)
-
-        movies = []
-        for m in movies_raw:
-            movies.append(m.to_dict({
-                'category': {},
-                'profile': {'types': {'quality': {}}},
-                'releases': {'status': {}, 'quality': {}},
-                'library': {'titles': {}, 'files': {}},
-                'files': {},
-            }))
 
+        total = db.count(db.get_many, 'movie_status', 'active')
         self.in_progress = {
-            'total': len(movies),
+            'total': total,
-            'to_go': len(movies),
+            'to_go': total,
         }
 
         try:
@@ -99,13 +88,15 @@ class MovieSearcher(SearcherBase, MovieTypeBase):
 
             for movie in movies:
 
+                movie_dict = db.run('media', 'to_dict', movie['_id'])
+
                 try:
-                    self.single(movie, search_protocols)
+                    self.single(movie_dict, search_protocols)
                 except IndexError:
-                    log.error('Forcing library update for %s, if you see this often, please report: %s', (movie['library']['identifier'], traceback.format_exc()))
+                    log.error('Forcing library update for %s, if you see this often, please report: %s', (movie['identifier'], traceback.format_exc()))
-                    fireEvent('library.update.movie', movie['library']['identifier'])
+                    fireEvent('library.update.movie', movie['identifier'])
                 except:
-                    log.error('Search failed for %s: %s', (movie['library']['identifier'], traceback.format_exc()))
+                    log.error('Search failed for %s: %s', (movie['identifier'], traceback.format_exc()))
 
                 self.in_progress['to_go'] -= 1
 
@@ -117,7 +108,6 @@ class MovieSearcher(SearcherBase, MovieTypeBase):
             pass
 
         self.in_progress = False
-        db.close()
 
     def single(self, movie, search_protocols = None, manual = False):
 
@@ -132,32 +122,30 @@ class MovieSearcher(SearcherBase, MovieTypeBase):
         except SearchSetupError:
             return
 
-        done_status = fireEvent('status.get', 'done', single = True)
+        if not movie['profile_id'] or (movie['status'] == 'done' and not manual):
-
-        if not movie['profile'] or (movie['status_id'] == done_status.get('id') and not manual):
             log.debug('Movie doesn\'t have a profile or already done, assuming in manage tab.')
             return
 
         pre_releases = fireEvent('quality.pre_releases', single = True)
         release_dates = fireEvent('library.update.movie.release_date', identifier = movie['library']['identifier'], merge = True)
-        available_status, ignored_status, failed_status = fireEvent('status.get', ['available', 'ignored', 'failed'], single = True)
 
         found_releases = []
         too_early_to_search = []
 
-        default_title = getTitle(movie['library'])
+        default_title = getTitle(movie)
         if not default_title:
             log.error('No proper info found for movie, removing it from library to cause it from having more issues.')
-            fireEvent('media.delete', movie['id'], single = True)
+            fireEvent('media.delete', movie['_id'], single = True)
             return
 
-        fireEvent('notify.frontend', type = 'movie.searcher.started', data = {'id': movie['id']}, message = 'Searching for "%s"' % default_title)
+        fireEvent('notify.frontend', type = 'movie.searcher.started', data = {'id': movie['_id']}, message = 'Searching for "%s"' % default_title)
 
-        db = get_session()
+        db = get_db()
 
         ret = False
+
         for quality_type in movie['profile']['types']:
-            if not self.conf('always_search') and not self.couldBeReleased(quality_type['quality']['identifier'] in pre_releases, release_dates, movie['library']['year']):
+            if not self.conf('always_search') and not self.couldBeReleased(quality_type['quality']['identifier'] in pre_releases, release_dates, movie['year']):
                 too_early_to_search.append(quality_type['quality']['identifier'])
                 continue
 
@@ -165,7 +153,7 @@ class MovieSearcher(SearcherBase, MovieTypeBase):
 
             # See if better quality is available
             for release in movie['releases']:
-                if release['quality']['order'] <= quality_type['quality']['order'] and release['status_id'] not in [available_status.get('id'), ignored_status.get('id'), failed_status.get('id')]:
+                if release['quality']['order'] <= quality_type['quality']['order'] and release['status'] not in ['available', 'ignored', 'failed']:
                     has_better_quality += 1
 
             # Don't search for quality lower then already available.
@@ -179,7 +167,7 @@ class MovieSearcher(SearcherBase, MovieTypeBase):
                     log.debug('Nothing found for %s in %s', (default_title, quality_type['quality']['label']))
 
                 # Check if movie isn't deleted while searching
-                if not db.query(Media).filter_by(id = movie.get('id')).first():
+                if not fireEvent('media.get', movie.get('_id'), single = True):
                     break
 
                 # Add them to this movie releases list
@@ -191,12 +179,12 @@ class MovieSearcher(SearcherBase, MovieTypeBase):
 
                 # Remove releases that aren't found anymore
                 for release in movie.get('releases', []):
-                    if release.get('status_id') == available_status.get('id') and release.get('identifier') not in found_releases:
+                    if release.get('status') == 'available' and release.get('identifier') not in found_releases:
-                        fireEvent('release.delete', release.get('id'), single = True)
+                        fireEvent('release.delete', release.get('_id'), single = True)
 
             else:
                 log.info('Better quality (%s) already available or snatched for %s', (quality_type['quality']['label'], default_title))
-                fireEvent('media.restatus', movie['id'])
+                fireEvent('media.restatus', movie['_id'])
                 break
 
             # Break if CP wants to shut down
@@ -206,9 +194,9 @@ class MovieSearcher(SearcherBase, MovieTypeBase):
         if len(too_early_to_search) > 0:
             log.info2('Too early to search for %s, %s', (too_early_to_search, default_title))
 
-        fireEvent('notify.frontend', type = 'movie.searcher.ended', data = {'id': movie['id']})
+        fireEvent('notify.frontend', type = 'movie.searcher.ended', data = {'id': movie['_id']})
 
-        db.close()
+        pass  #db.close()
         return ret
 
     def correctRelease(self, nzb = None, media = None, quality = None, **kwargs):
@@ -329,31 +317,23 @@ class MovieSearcher(SearcherBase, MovieTypeBase):
 
     def tryNextRelease(self, media_id, manual = False):
 
-        snatched_status, done_status, ignored_status = fireEvent('status.get', ['snatched', 'done', 'ignored'], single = True)
-
         try:
-            db = get_session()
+            db = get_db()
-            rels = db.query(Release) \
+            rels = db.run('media', 'with_status', media_id, status = ['snatched', 'done'])
-                .filter_by(movie_id = media_id) \
-                .filter(Release.status_id.in_([snatched_status.get('id'), done_status.get('id')])) \
-                .all()
 
             for rel in rels:
-                rel.status_id = ignored_status.get('id')
+                rel['status'] = 'ignored'
-            db.commit()
+                db.update(rel)
 
-            movie_dict = fireEvent('media.get', media_id = media_id, single = True)
+            movie_dict = db.run('media', 'to_dict', media_id)
-            log.info('Trying next release for: %s', getTitle(movie_dict['library']))
+            log.info('Trying next release for: %s', getTitle(movie_dict))
-            fireEvent('movie.searcher.single', movie_dict, manual = manual)
+            self.single(movie_dict, manual = manual)
 
             return True
 
         except:
             log.error('Failed searching for next release: %s', traceback.format_exc())
-            db.rollback()
             return False
-        finally:
-            db.close()
 
     def getSearchTitle(self, media):
         if media['type'] == 'movie':

couchpotato/core/media/movie/suggestion/main.py

@@ -33,7 +33,7 @@ class Suggestion(Plugin):
                     .options(joinedload_all('library')) \
                     .filter(or_(*[Media.status.has(identifier = s) for s in ['active', 'done']])).all()
                 movies = [x.library.identifier for x in active_movies]
-                db.close()
+                pass  #db.close()
 
             if not ignored or len(ignored) == 0:
                 ignored = splitString(Env.prop('suggest_ignore', default = ''))
@@ -98,7 +98,7 @@ class Suggestion(Plugin):
                 .filter(Media.status_id.in_([active_status.get('id'), done_status.get('id')])).all()
             movies = [x[0] for x in active_movies]
             movies.extend(seen)
-            db.close()
+            pass  #db.close()
 
             ignored.extend([x.get('imdb') for x in cached_suggestion])
             suggestions = fireEvent('movie.suggest', movies = movies, ignore = removeDuplicate(ignored), single = True)

couchpotato/core/notifications/core/index.py

@@ -0,0 +1,42 @@
+import time
+from CodernityDB.tree_index import TreeBasedIndex
+
+
+class NotificationIndex(TreeBasedIndex):
+
+    custom_header = """from CodernityDB.tree_index import TreeBasedIndex
+import time"""
+
+    def __init__(self, *args, **kwargs):
+        kwargs['key_format'] = 'I'
+        super(NotificationIndex, self).__init__(*args, **kwargs)
+
+    def make_key(self, key):
+        return key
+
+    def make_key_value(self, data):
+        if data.get('type') == 'notification':
+            added = data.get('added', time.time())
+            data['added'] = added
+
+            return added, None
+
+
+class NotificationUnreadIndex(TreeBasedIndex):
+
+    custom_header = """from CodernityDB.tree_index import TreeBasedIndex
+import time"""
+
+    def __init__(self, *args, **kwargs):
+        kwargs['key_format'] = 'I'
+        super(NotificationUnreadIndex, self).__init__(*args, **kwargs)
+
+    def make_key(self, key):
+        return key
+
+    def make_key_value(self, data):
+        if data.get('type') == 'notification' and not data.get('read'):
+            added = data.get('added', time.time())
+            data['added'] = added
+
+            return added, None

couchpotato/core/notifications/core/main.py

@@ -1,14 +1,13 @@
-from couchpotato import get_session
+from couchpotato import get_db
 from couchpotato.api import addApiView, addNonBlockApiView
 from couchpotato.core.event import addEvent, fireEvent
 from couchpotato.core.helpers.encoding import toUnicode
 from couchpotato.core.helpers.variable import tryInt, splitString
 from couchpotato.core.logger import CPLog
 from couchpotato.core.notifications.base import Notification
-from couchpotato.core.settings.model import Notification as Notif
+from .index import NotificationIndex, NotificationUnreadIndex
 from couchpotato.environment import Env
 from operator import itemgetter
-from sqlalchemy.sql.expression import or_
 import threading
 import time
 import traceback
@@ -58,48 +57,55 @@ class CoreNotifier(Notification):
         fireEvent('schedule.interval', 'core.check_messages', self.checkMessages, hours = 12, single = True)
         fireEvent('schedule.interval', 'core.clean_messages', self.cleanMessages, seconds = 15, single = True)
 
-        addEvent('app.load', self.clean)
+        addEvent('app.load2', self.clean)
-        addEvent('app.load', self.checkMessages)
+        addEvent('app.load2', self.checkMessages)
+
+        addEvent('database.setup', self.databaseSetup)
 
         self.messages = []
         self.listeners = []
         self.m_lock = threading.Lock()
 
-    def clean(self):
+    def databaseSetup(self):
+
+        db = get_db()
 
         try:
-            db = get_session()
+            db.add_index(NotificationIndex(db.path, 'notification'))
-            db.query(Notif).filter(Notif.added <= (int(time.time()) - 2419200)).delete()
+        except:
-            db.commit()
+            log.debug('Index already exists')
+            db.edit_index(NotificationIndex(db.path, 'notification'))
+
+        try:
+            db.add_index(NotificationUnreadIndex(db.path, 'notification_unread'))
+        except:
+            log.debug('Index already exists')
+            db.edit_index(NotificationUnreadIndex(db.path, 'notification_unread'))
+
+    def clean(self):
+        try:
+            db = get_db()
+            for n in db.all('notification', with_doc = True):
+                if n['doc']['added'] <= (int(time.time()) - 2419200):
+                    db.delete(n['doc'])
         except:
             log.error('Failed cleaning notification: %s', traceback.format_exc())
-            db.rollback()
-        finally:
-            db.close()
 
     def markAsRead(self, ids = None, **kwargs):
 
         ids = splitString(ids) if ids else None
 
         try:
-            db = get_session()
+            db = get_db()
-
+            for x in db.all('notification_unread', with_doc = True):
-            if ids:
+                if not ids or x['_id'] in ids:
-                q = db.query(Notif).filter(or_(*[Notif.id == tryInt(s) for s in ids]))
+                    x['doc']['read'] = True
-            else:
+                    db.update(x['doc'])
-                q = db.query(Notif).filter_by(read = False)
-
-            q.update({Notif.read: True})
-            db.commit()
-
             return {
                 'success': True
             }
         except:
             log.error('Failed mark as read: %s', traceback.format_exc())
-            db.rollback()
-        finally:
-            db.close()
 
         return {
             'success': False
@@ -107,26 +113,20 @@ class CoreNotifier(Notification):
 
     def listView(self, limit_offset = None, **kwargs):
 
-        db = get_session()
+        db = get_db()
-
-        q = db.query(Notif)
 
         if limit_offset:
             splt = splitString(limit_offset)
             limit = splt[0]
             offset = 0 if len(splt) is 1 else splt[1]
-            q = q.limit(limit).offset(offset)
+            results = db.get_many('notification', limit = limit, offset = offset, with_doc = True)
         else:
-            q = q.limit(200)
+            results = db.get_many('notification', limit = 200, with_doc = True)
 
-        results = q.all()
         notifications = []
         for n in results:
-            ndict = n.to_dict()
+            notifications.append(n['doc'])
-            ndict['type'] = 'notification'
-            notifications.append(ndict)
 
-        db.close()
         return {
             'success': True,
             'empty': len(notifications) == 0,
@@ -156,30 +156,23 @@ class CoreNotifier(Notification):
         if not data: data = {}
 
         try:
-            db = get_session()
+            db = get_db()
 
             data['notification_type'] = listener if listener else 'unknown'
 
-            n = Notif(
+            n = {
-                message = toUnicode(message),
+                'type': 'notification',
-                data = data
+                'time': time.time(),
-            )
+                'message': toUnicode(message),
-            db.add(n)
+                'data': data
-            db.commit()
+            }
+            db.insert(n)
 
-            ndict = n.to_dict()
+            self.frontend(type = listener, data = n)
-            ndict['type'] = 'notification'
-            ndict['time'] = time.time()
 
-            self.frontend(type = listener, data = data)
-
-            db.close()
             return True
         except:
             log.error('Failed notify: %s', traceback.format_exc())
-            db.rollback()
-        finally:
-            db.close()
 
     def frontend(self, type = 'notification', data = None, message = None):
         if not data: data = {}
@@ -278,18 +271,13 @@ class CoreNotifier(Notification):
 
         # Get unread
         if init:
-            db = get_session()
+            db = get_db()
 
-            notifications = db.query(Notif) \
+            notifications = db.all('notification_unread', with_doc = True)
-                .filter(or_(Notif.read == False, Notif.added > (time.time() - 259200))) \
-                .all()
 
             for n in notifications:
-                ndict = n.to_dict()
+                if n['doc'].get('added') > (time.time() - 259200):
-                ndict['type'] = 'notification'
+                    messages.append(n['doc'])
-                messages.append(ndict)
-
-            db.close()
 
         return {
             'success': True,

couchpotato/core/notifications/growl/main.py

@@ -16,7 +16,7 @@ class Growl(Notification):
         super(Growl, self).__init__()
 
         if self.isEnabled():
-            addEvent('app.load', self.register)
+            addEvent('app.load2', self.register)
 
     def register(self):
         if self.registered: return

couchpotato/core/plugins/automation/main.py

@@ -10,10 +10,10 @@ class Automation(Plugin):
 
     def __init__(self):
 
-        addEvent('app.load', self.setCrons)
+        addEvent('app.load2', self.setCrons)
 
         if not Env.get('dev'):
-            addEvent('app.load', self.addMovies)
+            addEvent('app.load2', self.addMovies)
 
         addEvent('setting.save.automation.hour.after', self.setCrons)
 

couchpotato/core/plugins/category/main.py

@@ -42,7 +42,7 @@ class CategoryPlugin(Plugin):
         for category in categories:
             temp.append(category.to_dict())
 
-        db.close()
+        pass  #db.close()
         return temp
 
     def save(self, **kwargs):
@@ -66,7 +66,7 @@ class CategoryPlugin(Plugin):
 
             category_dict = c.to_dict()
 
-            db.close()
+            pass  #db.close()
             return {
                 'success': True,
                 'category': category_dict
@@ -75,7 +75,7 @@ class CategoryPlugin(Plugin):
             log.error('Failed: %s', traceback.format_exc())
             db.rollback()
         finally:
-            db.close()
+            pass  #db.close()
 
         return {
             'success': False,
@@ -96,7 +96,7 @@ class CategoryPlugin(Plugin):
 
             db.commit()
 
-            db.close()
+            pass  #db.close()
             return {
                 'success': True
             }
@@ -104,7 +104,7 @@ class CategoryPlugin(Plugin):
             log.error('Failed: %s', traceback.format_exc())
             db.rollback()
         finally:
-            db.close()
+            pass  #db.close()
 
         return {
             'success': False
@@ -129,7 +129,7 @@ class CategoryPlugin(Plugin):
             except Exception as e:
                 message = log.error('Failed deleting category: %s', e)
 
-            db.close()
+            pass  #db.close()
             return {
                 'success': success,
                 'message': message
@@ -138,7 +138,7 @@ class CategoryPlugin(Plugin):
             log.error('Failed: %s', traceback.format_exc())
             db.rollback()
         finally:
-            db.close()
+            pass  #db.close()
 
         return {
             'success': False
@@ -158,4 +158,4 @@ class CategoryPlugin(Plugin):
             log.error('Failed: %s', traceback.format_exc())
             db.rollback()
         finally:
-            db.close()
+            pass  #db.close()

couchpotato/core/plugins/custom/main.py

@@ -10,7 +10,7 @@ log = CPLog(__name__)
 class Custom(Plugin):
 
     def __init__(self):
-        addEvent('app.load', self.createStructure)
+        addEvent('app.load2', self.createStructure)
 
     def createStructure(self):
 

couchpotato/core/plugins/dashboard/main.py

@@ -119,7 +119,7 @@ class Dashboard(Plugin):
                         'files': {},
                     }))
 
-        db.close()
+        pass  #db.close()
         return {
             'success': True,
             'empty': len(movies) == 0,

couchpotato/core/plugins/file/main.py

@@ -19,9 +19,7 @@ log = CPLog(__name__)
 class FileManager(Plugin):
 
     def __init__(self):
-        addEvent('file.add', self.add)
         addEvent('file.download', self.download)
-        addEvent('file.types', self.getTypes)
 
         addApiView('file.cache/(.*)', self.showCacheFile, static = True, docs = {
             'desc': 'Return a file from the cp_data/cache directory',
@@ -31,36 +29,8 @@ class FileManager(Plugin):
             'return': {'type': 'file'}
         })
 
-        addApiView('file.types', self.getTypesView, docs = {
-            'desc': 'Return a list of all the file types and their ids.',
-            'return': {'type': 'object', 'example': """{
-    'types': [
-        {
-            "identifier": "poster_original",
-            "type": "image",
-            "id": 1,
-            "name": "Poster_original"
-        },
-        {
-            "identifier": "poster",
-            "type": "image",
-            "id": 2,
-            "name": "Poster"
-        },
-        etc
-    ]
-}"""}
-        })
-
-        addEvent('app.load', self.cleanup)
-        addEvent('app.load', self.init)
-
-    def init(self):
-
-        for type_tuple in Scanner.file_types.values():
-            self.getType(type_tuple)
-
     def cleanup(self):
+        # TODO: unused
 
         # Wait a bit after starting before cleanup
         time.sleep(3)
@@ -77,9 +47,6 @@ class FileManager(Plugin):
                             os.remove(file_path)
         except:
             log.error('Failed removing unused file: %s', traceback.format_exc())
-            db.rollback()
-        finally:
-            db.close()
 
     def showCacheFile(self, route, **kwargs):
         Env.get('app').add_handlers(".*$", [('%s%s' % (Env.get('api_base'), route), StaticFileHandler, {'path': Env.get('cache_dir')})])
@@ -101,76 +68,3 @@ class FileManager(Plugin):
 
         self.createFile(dest, filedata, binary = True)
         return dest
-
-    def add(self, path = '', part = 1, type_tuple = (), available = 1, properties = None):
-        if not properties: properties = {}
-
-        try:
-            db = get_session()
-            type_id = self.getType(type_tuple).get('id')
-
-            f = db.query(File).filter(File.path == toUnicode(path)).first()
-            if not f:
-                f = File()
-                db.add(f)
-
-            f.path = toUnicode(path)
-            f.part = part
-            f.available = available
-            f.type_id = type_id
-
-            db.commit()
-
-            file_dict = f.to_dict()
-
-            return file_dict
-        except:
-            log.error('Failed adding file: %s, %s', (path, traceback.format_exc()))
-            db.rollback()
-        finally:
-            db.close()
-
-    def getType(self, type_tuple):
-
-        try:
-            db = get_session()
-            type_type, type_identifier = type_tuple
-
-            ft = db.query(FileType).filter_by(identifier = type_identifier).first()
-            if not ft:
-                ft = FileType(
-                    type = toUnicode(type_type),
-                    identifier = type_identifier,
-                    name = toUnicode(type_identifier[0].capitalize() + type_identifier[1:])
-                )
-                db.add(ft)
-                db.commit()
-
-            type_dict = ft.to_dict()
-
-            return type_dict
-        except:
-            log.error('Failed getting type: %s, %s', (type_tuple, traceback.format_exc()))
-            db.rollback()
-        finally:
-            db.close()
-
-
-    def getTypes(self):
-
-        db = get_session()
-
-        results = db.query(FileType).all()
-
-        types = []
-        for type_object in results:
-            types.append(type_object.to_dict())
-
-        db.close()
-        return types
-
-    def getTypesView(self, **kwargs):
-
-        return {
-            'types': self.getTypes()
-        }

couchpotato/core/plugins/manage/main.py

@@ -47,7 +47,7 @@ class Manage(Plugin):
         })
 
         if not Env.get('dev') and self.conf('startup_scan'):
-            addEvent('app.load', self.updateLibraryQuick)
+            addEvent('app.load2', self.updateLibraryQuick)
 
     def getProgress(self, **kwargs):
         return {

couchpotato/core/plugins/nosql/__init__.py

@@ -0,0 +1,7 @@
+from .main import NoSQL
+
+
+def start():
+    return NoSQL()
+
+config = []

couchpotato/core/plugins/nosql/index.py

@@ -0,0 +1,20 @@
+from hashlib import md5
+from CodernityDB.tree_index import TreeBasedIndex
+
+
+class ReleaseIndex(TreeBasedIndex):
+
+    def __init__(self, *args, **kwargs):
+        kwargs['key_format'] = '16s'
+        super(ReleaseIndex, self).__init__(*args, **kwargs)
+
+    def make_key(self, key):
+        return md5(key).digest()
+
+    def make_key_value(self, data):
+        if data.get('type') == 'release':
+            return md5(data['media_id']).digest(), None
+
+    def run_for_media(self, db, media_id):
+        for release in db.get_many('release', media_id, with_doc=True):
+            yield release['doc']

couchpotato/core/plugins/nosql/main.py

@@ -0,0 +1,72 @@
+import time
+from couchpotato import CPLog
+from couchpotato.core.plugins.base import Plugin
+from .index import ReleaseIndex, MediaIMDBIndex, TitleIndex
+
+log = CPLog(__name__)
+
+
+class NoSQL(Plugin):
+
+    db = None
+
+    def test(self):
+
+        db = self.db
+
+        try: db.add_index(ReleaseIndex(db.path, 'release'))
+        except: log.debug('Index already exists')
+
+        for id in range(10):
+            media = db.insert({
+                'type': 'media',
+                'tmdb': id,
+                'imdb': 'tt%s' % id,
+                'last_edit': 0,
+                'status': 'active',
+                'title': 'Lord of the Rings: The Return of the King',
+                'year': 2011,
+                'profile_id': 0,
+                'category_id': 0,
+            })
+
+            db.insert({
+                'media_id': media['_id'],
+                'type': 'title',
+                'title': 'Lord of the Rings: The Return of the King',
+            })
+
+            for x in range(40):
+                db.insert({
+                    'media_id': media['_id'],
+                    'type': 'release',
+                    'name': 'Release %s' % x
+                })
+
+        print db.count(db.all, 'media')
+
+        m = db.get('media', 'tt0')
+        db.get('id', m['_id'])
+
+        start = time.time()
+        print list(db.get_many('media_title', 'lord of'))
+        print time.time() - start
+
+        return
+
+        return
+
+        for media in db.all('media', with_doc = True):
+            doc = media['doc']
+            for r in db.run('release', 'for_media', media['_id']):
+                db.delete(r)
+
+            db.delete(doc)
+            break
+
+        start = time.time()
+        db.reindex()
+        print time.time() - start
+
+        print db.count(db.all, 'media')
+        print db.count(db.all, 'release')

couchpotato/core/plugins/profile/main.py

@@ -1,5 +1,5 @@
 import traceback
-from couchpotato import get_session
+from couchpotato import get_session, get_db
 from couchpotato.api import addApiView
 from couchpotato.core.event import addEvent, fireEvent
 from couchpotato.core.helpers.encoding import toUnicode
@@ -31,27 +31,24 @@ class ProfilePlugin(Plugin):
         })
 
         addEvent('app.initialize', self.fill, priority = 90)
-        addEvent('app.load', self.forceDefaults)
+        addEvent('app.load2', self.forceDefaults)
 
     def forceDefaults(self):
 
         # Get all active movies without profile
-        active_status = fireEvent('status.get', 'active', single = True)
-
         try:
-            db = get_session()
+            db = get_db()
-            movies = db.query(Media).filter(Media.status_id == active_status.get('id'), Media.profile == None).all()
+            medias = db.run('media', 'with_status', ['active'])
 
-            if len(movies) > 0:
+            profile_ids = [x.get('_id') for x in self.all()]
-                default_profile = self.default()
+
-                for movie in movies:
+            for media in medias:
-                    movie.profile_id = default_profile.get('id')
+                if media['profile_id'] not in profile_ids:
-                    db.commit()
+                    default_profile = self.default()
+                    media['profile_id'] = default_profile.get('id')
+                    db.update(media)
         except:
             log.error('Failed: %s', traceback.format_exc())
-            db.rollback()
-        finally:
-            db.close()
 
     def allView(self, **kwargs):
 
@@ -62,17 +59,10 @@ class ProfilePlugin(Plugin):
 
     def all(self):
 
-        db = get_session()
+        db = get_db()
-        profiles = db.query(Profile) \
+        profiles = db.all('profile', with_doc = True)
-            .options(joinedload_all('types')) \
-            .all()
 
-        temp = []
+        return list(profiles)
-        for profile in profiles:
-            temp.append(profile.to_dict(self.to_dict))
-
-        db.close()
-        return temp
 
     def save(self, **kwargs):
 
@@ -115,7 +105,7 @@ class ProfilePlugin(Plugin):
             log.error('Failed: %s', traceback.format_exc())
             db.rollback()
         finally:
-            db.close()
+            pass  #db.close()
 
         return {
             'success': False
@@ -128,7 +118,7 @@ class ProfilePlugin(Plugin):
             .options(joinedload_all('types')) \
             .first()
         default_dict = default.to_dict(self.to_dict)
-        db.close()
+        pass  #db.close()
 
         return default_dict
 
@@ -154,7 +144,7 @@ class ProfilePlugin(Plugin):
             log.error('Failed: %s', traceback.format_exc())
             db.rollback()
         finally:
-            db.close()
+            pass  #db.close()
 
         return {
             'success': False
@@ -188,7 +178,7 @@ class ProfilePlugin(Plugin):
             log.error('Failed: %s', traceback.format_exc())
             db.rollback()
         finally:
-            db.close()
+            pass  #db.close()
 
         return {
             'success': False
@@ -243,6 +233,6 @@ class ProfilePlugin(Plugin):
             log.error('Failed: %s', traceback.format_exc())
             db.rollback()
         finally:
-            db.close()
+            pass  #db.close()
 
         return False

couchpotato/core/plugins/quality/main.py

@@ -79,7 +79,7 @@ class QualityPlugin(Plugin):
 
         self.cached_qualities = temp
 
-        db.close()
+        pass  #db.close()
         return temp
 
     def single(self, identifier = ''):
@@ -91,7 +91,7 @@ class QualityPlugin(Plugin):
         if quality:
             quality_dict = dict(self.getQuality(quality.identifier), **quality.to_dict())
 
-        db.close()
+        pass  #db.close()
         return quality_dict
 
     def getQuality(self, identifier):
@@ -119,7 +119,7 @@ class QualityPlugin(Plugin):
             log.error('Failed: %s', traceback.format_exc())
             db.rollback()
         finally:
-            db.close()
+            pass  #db.close()
 
         return {
             'success': False
@@ -181,7 +181,7 @@ class QualityPlugin(Plugin):
             log.error('Failed: %s', traceback.format_exc())
             db.rollback()
         finally:
-            db.close()
+            pass  #db.close()
 
         return False
 

couchpotato/core/plugins/release/main.py

@@ -1,4 +1,4 @@
-from couchpotato import get_session, md5
+from couchpotato import get_session, md5, get_db
 from couchpotato.api import addApiView
 from couchpotato.core.event import fireEvent, addEvent
 from couchpotato.core.helpers.encoding import ss, toUnicode
@@ -72,34 +72,32 @@ class Release(Plugin):
         done_status, available_status, snatched_status, downloaded_status, ignored_status = \
             fireEvent('status.get', ['done', 'available', 'snatched', 'downloaded', 'ignored'], single = True)
 
-        db = get_session()
+        db = get_db()
 
         # get movies last_edit more than a week ago
-        media = db.query(Media) \
+        medias = db.run('media', 'with_status', ['done'])
-            .filter(Media.status_id == done_status.get('id'), Media.last_edit < (now - week)) \
-            .all()
 
-        for item in media:
+        for media in medias:
-            for rel in item.releases:
+            if media['last_edit'] > (now - week):
+                continue
+
+            for rel in db.run('release', 'for_media', media['_id']):
                 # Remove all available releases
-                if rel.status_id in [available_status.get('id')]:
+                if rel['status'] in ['available']:
-                    fireEvent('release.delete', id = rel.id, single = True)
+                    fireEvent('release.delete', id = rel['_id'], single = True)
-                # Set all snatched and downloaded releases to ignored to make sure they are ignored when re-adding the move
-                elif rel.status_id in [snatched_status.get('id'), downloaded_status.get('id')]:
-                    self.updateStatus(id = rel.id, status = ignored_status)
 
-        db.close()
+                # Set all snatched and downloaded releases to ignored to make sure they are ignored when re-adding the move
+                elif rel['status'] in ['snatched', 'downloaded']:
+                    self.updateStatus(id = rel['id'], status = ignored_status)
 
 
     def add(self, group):
 
         try:
-            db = get_session()
+            db = get_db()
 
             identifier = '%s.%s.%s' % (group['library']['identifier'], group['meta_data'].get('audio', 'unknown'), group['meta_data']['quality']['identifier'])
 
-            done_status, snatched_status = fireEvent('status.get', ['done', 'snatched'], single = True)
-
             # Add movie
             media = db.query(Media).filter_by(library_id = group['library'].get('id')).first()
             if not media:
@@ -129,41 +127,28 @@ class Release(Plugin):
                 db.commit()
 
             # Add each file type
-            added_files = []
+            rel['files'] = []
             for type in group['files']:
                 for cur_file in group['files'][type]:
-                    added_file = self.saveFile(cur_file, type = type, include_media_info = type is 'movie')
+                    added_file = self.saveFile(cur_file, type = type)
-                    added_files.append(added_file.get('id'))
+                    rel['files'].append(added_file.get('id'))
 
-            # Add the release files in batch
+            fireEvent('media.restatus', media['_id'])
-            try:
-                added_files = db.query(File).filter(or_(*[File.id == x for x in added_files])).all()
-                rel.files.extend(added_files)
-                db.commit()
-            except:
-                log.debug('Failed to attach "%s" to release: %s', (added_files, traceback.format_exc()))
-
-            fireEvent('media.restatus', media.id)
 
             return True
         except:
             log.error('Failed: %s', traceback.format_exc())
-            db.rollback()
-        finally:
-            db.close()
 
         return False
 
     def saveFile(self, filepath, type = 'unknown', include_media_info = False):
 
-        properties = {}
-
-        # Get media info for files
-        if include_media_info:
-            properties = {}
-
         # Check database and update/insert if necessary
-        return fireEvent('file.add', path = filepath, part = fireEvent('scanner.partnumber', file, single = True), type_tuple = Scanner.file_types.get(type), properties = properties, single = True)
+        return {
+            'type': '%s_%s' % Scanner.file_types.get(type),
+            'path': filepath,
+            'part': fireEvent('scanner.partnumber', file, single = True),
+        }
 
     def deleteView(self, id = None, **kwargs):
 
@@ -171,60 +156,58 @@ class Release(Plugin):
             'success': self.delete(id)
         }
 
-    def delete(self, id):
+    def delete(self, release_id):
 
         try:
-            db = get_session()
+            db = get_db()
 
-            rel = db.query(Relea).filter_by(id = id).first()
+            rel = db.get('release', release_id, with_doc = True)
-            if rel:
+            db.delete(rel['doc'])
-                db.delete(rel)
+            return True
-                db.commit()
-                return True
         except:
             log.error('Failed: %s', traceback.format_exc())
-            db.rollback()
-        finally:
-            db.close()
 
         return False
 
-    def clean(self, id):
+    def clean(self, release_id):
 
         try:
-            db = get_session()
+            db = get_db()
 
-            rel = db.query(Relea).filter_by(id = id).first()
+            rel = db.get('release', release_id, with_doc = True)
-            if rel:
+            files = []
-                for release_file in rel.files:
+            for release_file in rel['files']:
-                    if not os.path.isfile(ss(release_file.path)):
+                if os.path.isfile(ss(release_file['path'])):
-                        db.delete(release_file)
+                    files.append(release_file)
-                db.commit()
 
-                if len(rel.files) == 0:
+            if len(rel['files']) == 0:
-                    self.delete(id)
+                self.delete(rel['_id'])
+            else:
+                rel['files'] = files
+                db.update(rel)
 
-                return True
+            return True
         except:
             log.error('Failed: %s', traceback.format_exc())
-            db.rollback()
-        finally:
-            db.close()
 
         return False
 
-    def ignore(self, id = None, **kwargs):
+    def ignore(self, release_id = None, **kwargs):
 
-        db = get_session()
+        db = get_db()
 
-        rel = db.query(Relea).filter_by(id = id).first()
+        try:
-        if rel:
+            rel = db.get('release', release_id, with_doc = True)['doc']
-            ignored_status, failed_status, available_status = fireEvent('status.get', ['ignored', 'failed', 'available'], single = True)
+            self.updateStatus(release_id, 'available' if rel['status'] in ['ignored', 'failed'] else 'ignored')
-            self.updateStatus(id, available_status if rel.status_id in [ignored_status.get('id'), failed_status.get('id')] else ignored_status)
+
+            return {
+                'success': True
+            }
+        except:
+            log.error('Failed: %s', traceback.format_exc())
 
-        db.close()
         return {
-            'success': True
+            'success': False
         }
 
     def manualDownload(self, id = None, **kwargs):
@@ -265,7 +248,7 @@ class Release(Plugin):
         if success:
             fireEvent('notify.frontend', type = 'release.manual_download', data = True, message = 'Successfully snatched "%s"' % item['name'])
 
-        db.close()
+        pass  #db.close()
         return {
             'success': success == True
         }
@@ -361,7 +344,7 @@ class Release(Plugin):
             db.rollback()
             return False
         finally:
-            db.close()
+            pass  #db.close()
 
         return True
 
@@ -441,7 +424,7 @@ class Release(Plugin):
             log.error('Failed: %s', traceback.format_exc())
             db.rollback()
         finally:
-            db.close()
+            pass  #db.close()
 
         return []
 
@@ -458,7 +441,7 @@ class Release(Plugin):
         releases = [r.to_dict({'info': {}, 'files': {}}) for r in releases_raw]
         releases = sorted(releases, key = lambda k: k['info'].get('score', 0), reverse = True)
 
-        db.close()
+        pass  #db.close()
         return releases
 
     def forMovieView(self, id = None, **kwargs):
@@ -506,6 +489,6 @@ class Release(Plugin):
             log.error('Failed: %s', traceback.format_exc())
             db.rollback()
         finally:
-            db.close()
+            pass  #db.close()
 
         return False

couchpotato/core/plugins/renamer/main.py

@@ -598,7 +598,7 @@ class Renamer(Plugin):
                 break
 
         self.renaming_started = False
-        db.close()
+        pass  #db.close()
 
     def getRenameExtras(self, extra_type = '', replacements = None, folder_name = '', file_name = '', destination = '', group = None, current_file = '', remove_multiple = False):
         if not group: group = {}
@@ -1045,7 +1045,7 @@ Remove it if you want it to be renamed (again, or at least let it try again)
             self.checking_snatched = False
             return True
         finally:
-            db.close()
+            pass  #db.close()
 
     def extendReleaseDownload(self, release_download):
 
@@ -1068,7 +1068,7 @@ Remove it if you want it to be renamed (again, or at least let it try again)
             if not rls:
                 log.error('Download ID %s from downloader %s not found in releases', (release_download.get('id'), release_download.get('downloader')))
 
-            db.close()
+            pass  #db.close()
 
         if rls:
 

couchpotato/core/plugins/scanner/main.py

@@ -441,7 +441,7 @@ class Scanner(Plugin):
         else:
             log.debug('Found no movies in the folder %s', folder)
 
-        db.close()
+        pass  #db.close()
         return processed_movies
 
     def getMetaData(self, group, folder = '', release_download = None):
@@ -609,7 +609,7 @@ class Scanner(Plugin):
                     break
                 except:
                     pass
-            db.close()
+            pass  #db.close()
 
         # Search based on identifiers
         if not imdb_id:

couchpotato/core/plugins/status/__init__.py

@@ -1,7 +0,0 @@
-from .main import StatusPlugin
-
-
-def start():
-    return StatusPlugin()
-
-config = []

couchpotato/core/plugins/status/main.py

@@ -1,137 +0,0 @@
-import traceback
-from couchpotato import get_session
-from couchpotato.api import addApiView
-from couchpotato.core.event import addEvent
-from couchpotato.core.helpers.encoding import toUnicode
-from couchpotato.core.logger import CPLog
-from couchpotato.core.plugins.base import Plugin
-from couchpotato.core.settings.model import Status
-
-log = CPLog(__name__)
-
-
-class StatusPlugin(Plugin):
-
-    statuses = {
-        'needs_update': 'Needs update',
-        'active': 'Active',
-        'done': 'Done',
-        'downloaded': 'Downloaded',
-        'wanted': 'Wanted',
-        'snatched': 'Snatched',
-        'failed': 'Failed',
-        'deleted': 'Deleted',
-        'ignored': 'Ignored',
-        'available': 'Available',
-        'suggest': 'Suggest',
-        'seeding': 'Seeding',
-        'missing': 'Missing',
-    }
-    status_cached = {}
-
-    def __init__(self):
-        addEvent('status.get', self.get)
-        addEvent('status.get_by_id', self.getById)
-        addEvent('status.all', self.all)
-        addEvent('app.initialize', self.fill)
-        addEvent('app.load', self.all)  # Cache all statuses
-
-        addApiView('status.list', self.list, docs = {
-            'desc': 'Check for available update',
-            'return': {'type': 'object', 'example': """{
-            'success': True,
-            'list': array, statuses
-}"""}
-        })
-
-    def list(self, **kwargs):
-
-        return {
-            'success': True,
-            'list': self.all()
-        }
-
-    def getById(self, id):
-        db = get_session()
-        status = db.query(Status).filter_by(id = id).first()
-        status_dict = status.to_dict()
-        db.close()
-
-        return status_dict
-
-    def all(self):
-
-        db = get_session()
-
-        statuses = db.query(Status).all()
-
-        temp = []
-        for status in statuses:
-            s = status.to_dict()
-            temp.append(s)
-
-            # Update cache
-            self.status_cached[status.identifier] = s
-
-        db.close()
-        return temp
-
-    def get(self, identifiers):
-
-        if not isinstance(identifiers, list):
-            identifiers = [identifiers]
-
-        try:
-            db = get_session()
-            return_list = []
-
-            for identifier in identifiers:
-
-                if self.status_cached.get(identifier):
-                    return_list.append(self.status_cached.get(identifier))
-                    continue
-
-                s = db.query(Status).filter_by(identifier = identifier).first()
-                if not s:
-                    s = Status(
-                        identifier = identifier,
-                        label = toUnicode(identifier.capitalize())
-                    )
-                    db.add(s)
-                    db.commit()
-
-                status_dict = s.to_dict()
-
-                self.status_cached[identifier] = status_dict
-                return_list.append(status_dict)
-
-            return return_list if len(identifiers) > 1 else return_list[0]
-        except:
-            log.error('Failed: %s', traceback.format_exc())
-            db.rollback()
-        finally:
-            db.close()
-
-    def fill(self):
-
-        try:
-            db = get_session()
-
-            for identifier, label in self.statuses.items():
-                s = db.query(Status).filter_by(identifier = identifier).first()
-                if not s:
-                    log.info('Creating status: %s', label)
-                    s = Status(
-                        identifier = identifier,
-                        label = toUnicode(label)
-                    )
-                    db.add(s)
-
-                s.label = toUnicode(label)
-                db.commit()
-        except:
-            log.error('Failed: %s', traceback.format_exc())
-            db.rollback()
-        finally:
-            db.close()
-

couchpotato/core/plugins/status/static/status.js

@@ -1,17 +0,0 @@
-var StatusBase = new Class({
-
-	setup: function(statuses){
-		var self = this;
-
-		self.statuses = statuses;
-
-	},
-
-	get: function(id){
-		return this.statuses.filter(function(status){
-			return status.id == id
-		}).pick()
-	},
-
-});
-window.Status = new StatusBase();

couchpotato/core/plugins/subtitle/main.py

@@ -41,7 +41,7 @@ class Subtitle(Plugin):
                     # get subtitles for those files
                     subliminal.list_subtitles(files, cache_dir = Env.get('cache_dir'), multi = True, languages = self.getLanguages(), services = self.services)
 
-        db.close()
+        pass  #db.close()
 
     def searchSingle(self, group):
         if self.isDisabled(): return

couchpotato/core/providers/info/_modifier/main.py

@@ -104,7 +104,7 @@ class MovieResultModifier(Plugin):
         except:
             log.error('Tried getting more info on searched movies: %s', traceback.format_exc())
 
-        db.close()
+        pass  #db.close()
         return temp
 
     def checkLibrary(self, result):

couchpotato/core/providers/metadata/base.py

@@ -90,6 +90,7 @@ class MetaDataBase(Plugin):
         file_type = {}
 
         for ft in file_types:
+            # TODO: change type to "image_"+wanted_file_type
             if ft.get('identifier') == wanted_file_type:
                 file_type = ft
                 break

couchpotato/core/settings/__init__.py

@@ -4,6 +4,7 @@ from couchpotato.api import addApiView
 from couchpotato.core.event import addEvent, fireEvent
 from couchpotato.core.helpers.encoding import toUnicode
 from couchpotato.core.helpers.variable import mergeDicts, tryInt, tryFloat
+from couchpotato.core.settings.index import PropertyIndex
 from couchpotato.core.settings.model import Properties
 import ConfigParser
 
@@ -42,6 +43,7 @@ class Settings(object):
     }
 }"""}
         })
+
         addApiView('settings.save', self.saveView, docs = {
             'desc': 'Save setting to config file (settings.conf)',
             'params': {
@@ -51,6 +53,8 @@ class Settings(object):
             }
         })
 
+        addEvent('database.setup', self.databaseSetup)
+
     def setFile(self, config_file):
         self.file = config_file
 
@@ -62,6 +66,17 @@ class Settings(object):
 
         self.connectEvents()
 
+    def databaseSetup(self):
+        from couchpotato import get_db
+
+        db = get_db()
+
+        try:
+            db.add_index(PropertyIndex(db.path, 'property'))
+        except:
+            self.log.debug('Index already exists')
+            db.edit_index(PropertyIndex(db.path, 'property'))
+
     def parser(self):
         return self.p
 
@@ -206,36 +221,33 @@ class Settings(object):
         }
 
     def getProperty(self, identifier):
-        from couchpotato import get_session
+        from couchpotato import get_db
 
-        db = get_session()
+        db = get_db()
         prop = None
         try:
-            propert = db.query(Properties).filter_by(identifier = identifier).first()
+            propert = db.get('property', identifier, with_doc = True)
-            prop = propert.value
+            prop = propert['doc']['value']
         except:
-            pass
+            self.log.debug('Property doesn\'t exist: %s', traceback.format_exc(0))
 
-        db.close()
         return prop
 
     def setProperty(self, identifier, value = ''):
-        from couchpotato import get_session
+        from couchpotato import get_db
+
+        db = get_db()
 
         try:
-            db = get_session()
+            p = db.get('property', identifier, with_doc = True)
-
+            p['doc'].update({
-            p = db.query(Properties).filter_by(identifier = identifier).first()
+                'identifier': identifier,
-            if not p:
+                'value': toUnicode(value),
-                p = Properties()
+            })
-                db.add(p)
+            db.update(p['doc'])
-
-            p.identifier = identifier
-            p.value = toUnicode(value)
-
-            db.commit()
         except:
-            self.log.error('Failed: %s', traceback.format_exc())
+            db.insert({
-            db.rollback()
+                'type': 'property',
-        finally:
+                'identifier': identifier,
-            db.close()
+                'value': toUnicode(value),
+            })

couchpotato/core/settings/index.py

@@ -0,0 +1,16 @@
+from CodernityDB.hash_index import UniqueHashIndex, HashIndex
+from hashlib import md5
+
+
+class PropertyIndex(HashIndex):
+
+    def __init__(self, *args, **kwargs):
+        kwargs['key_format'] = '16s'
+        super(PropertyIndex, self).__init__(*args, **kwargs)
+
+    def make_key(self, key):
+        return md5(key).digest()
+
+    def make_key_value(self, data):
+        if data.get('type') == 'property':
+            return md5(data['identifier']).digest(), None

couchpotato/environment.py

@@ -31,7 +31,7 @@ class Env(object):
     _app_dir = ""
     _data_dir = ""
     _cache_dir = ""
-    _db_path = ""
+    _db = ""
     _log_path = ""
 
     @staticmethod
@@ -54,28 +54,6 @@ class Env(object):
     def set(attr, value):
         return setattr(Env, '_' + attr, value)
 
-    @staticmethod
-    def getSession():
-        existing_session = Env.get('session')
-        if existing_session:
-            return existing_session()
-
-        session = scoped_session(sessionmaker(bind = Env.getEngine()))
-        Env.set('session', session)
-
-        return session()
-
-    @staticmethod
-    def getEngine():
-        existing_engine = Env.get('engine')
-        if existing_engine:
-            return existing_engine
-
-        engine = create_engine(Env.get('db_path'), echo = False)
-        Env.set('engine', engine)
-
-        return engine
-
     @staticmethod
     def setting(attr, section = 'core', value = None, default = '', type = None):
 

couchpotato/runner.py

@@ -1,3 +1,4 @@
+from CodernityDB.database_thread_safe import ThreadSafeDatabase
 from argparse import ArgumentParser
 from cache import FileSystemCache
 from couchpotato import KeyHandler, LoginHandler, LogoutHandler
@@ -82,52 +83,25 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En
     Env.set('encoding', encoding)
 
     # Do db stuff
-    db_path = toUnicode(os.path.join(data_dir, 'couchpotato.db'))
+    db_path = toUnicode(os.path.join(data_dir, 'database'))
 
-    # Backup before start and cleanup old databases
+    # Check if database exists
-    new_backup = toUnicode(os.path.join(data_dir, 'db_backup', str(int(time.time()))))
+    print db_path
-    if not os.path.isdir(new_backup): os.makedirs(new_backup)
 
-    # Remove older backups, keep backups 3 days or at least 3
+    db = ThreadSafeDatabase(db_path)
-    backups = []
+    db_exists = db.exists()
-    for directory in os.listdir(os.path.dirname(new_backup)):
+    if db_exists:
-        backup = toUnicode(os.path.join(os.path.dirname(new_backup), directory))
+        db.open()
-        if os.path.isdir(backup):
+    else:
-            backups.append(backup)
+        db.create()
 
-    latest_backup = tryInt(os.path.basename(sorted(backups)[-1])) if len(backups) > 0 else 0
+    # TODO:Backup before start and cleanup old databases
-    if latest_backup < time.time() - 3600:
-        # Create path and copy
-        src_files = [options.config_file, db_path, db_path + '-shm', db_path + '-wal']
-        for src_file in src_files:
-            if os.path.isfile(src_file):
-                dst_file = toUnicode(os.path.join(new_backup, os.path.basename(src_file)))
-                shutil.copyfile(src_file, dst_file)
-
-                # Try and copy stats seperately
-                try: shutil.copystat(src_file, dst_file)
-                except: pass
-
-    total_backups = len(backups)
-    for backup in backups:
-        if total_backups > 3:
-            if tryInt(os.path.basename(backup)) < time.time() - 259200:
-                for the_file in os.listdir(backup):
-                    file_path = os.path.join(backup, the_file)
-                    try:
-                        if os.path.isfile(file_path):
-                            os.remove(file_path)
-                    except:
-                        raise
-
-                os.rmdir(backup)
-                total_backups -= 1
 
     # Register environment settings
     Env.set('app_dir', toUnicode(base_path))
     Env.set('data_dir', toUnicode(data_dir))
     Env.set('log_path', toUnicode(os.path.join(log_dir, 'CouchPotato.log')))
-    Env.set('db_path', toUnicode('sqlite:///' + db_path))
+    Env.set('db', db)
     Env.set('cache_dir', toUnicode(os.path.join(data_dir, 'cache')))
     Env.set('cache', FileSystemCache(toUnicode(os.path.join(Env.get('cache_dir'), 'python'))))
     Env.set('console_log', options.console_log)
@@ -183,34 +157,6 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En
         log.warning('%s %s %s line:%s', (category, message, filename, lineno))
     warnings.showwarning = customwarn
 
-    # Check if database exists
-    db = Env.get('db_path')
-    db_exists = os.path.isfile(toUnicode(db_path))
-
-    # Load migrations
-    if False and db_exists:
-
-        from migrate.versioning.api import version_control, db_version, version, upgrade
-        repo = os.path.join(base_path, 'couchpotato', 'core', 'migration')
-
-        latest_db_version = version(repo)
-        try:
-            current_db_version = db_version(db, repo)
-        except:
-            version_control(db, repo, version = latest_db_version)
-            current_db_version = db_version(db, repo)
-
-        if current_db_version < latest_db_version:
-            if development:
-                log.error('There is a database migration ready, but you are running development mode, so it won\'t be used. If you see this, you are stupid. Please disable development mode.')
-            else:
-                log.info('Doing database upgrade. From %d to %d', (current_db_version, latest_db_version))
-                upgrade(db, repo)
-
-    # Configure Database
-    from couchpotato.core.settings.model import setup
-    setup()
-
     # Create app
     from couchpotato import WebHandler
     web_base = ('/' + Env.setting('url_base').lstrip('/') + '/') if Env.setting('url_base') else '/'
@@ -279,6 +225,7 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En
     # Fill database with needed stuff
     if not db_exists:
         fireEvent('app.initialize', in_order = True)
+    fireEvent('database.setup')
 
     # Go go go!
     from tornado.ioloop import IOLoop
@@ -287,7 +234,7 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En
     # Some logging and fire load event
     try: log.info('Starting server on port %(port)s', config)
     except: pass
-    fireEventAsync('app.load')
+    fireEventAsync('app.load2')
 
     if config['ssl_cert'] and config['ssl_key']:
         server = HTTPServer(application, no_keep_alive = True, ssl_options = {

couchpotato/templates/index.html

@@ -71,8 +71,6 @@
 
 				Status.setup({{ json_encode(fireEvent('status.all', single = True)) }});
 
-				File.Type.setup({{ json_encode(fireEvent('file.types', single = True)) }});
-
 				CategoryList.setup({{ json_encode(fireEvent('category.all', single = True)) }});
 
 				App.setup({
@@ -95,4 +93,4 @@
 		<title>CouchPotato</title>
 	</head>
 	<body></body>
-</html>
+</html>

libs/CodernityDB/__init__.py

@@ -0,0 +1,20 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright 2011-2013 Codernity (http://codernity.com)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+__version__ = '0.4.2'
+__license__ = "Apache 2.0"

libs/CodernityDB/database_gevent.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright 2011-2013 Codernity (http://codernity.com)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from gevent.lock import RLock
+
+from CodernityDB.env import cdb_environment
+
+cdb_environment['mode'] = "gevent"
+cdb_environment['rlock_obj'] = RLock
+
+
+# from CodernityDB.database import Database
+from CodernityDB.database_safe_shared import SafeDatabase
+
+
+class GeventDatabase(SafeDatabase):
+    pass

libs/CodernityDB/database_safe_shared.py

@@ -0,0 +1,229 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright 2011-2013 Codernity (http://codernity.com)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from CodernityDB.env import cdb_environment
+from CodernityDB.database import PreconditionsException, RevConflict, Database
+# from database import Database
+
+from collections import defaultdict
+from functools import wraps
+from types import MethodType
+
+
+class th_safe_gen:
+
+    def __init__(self, name, gen, l=None):
+        self.lock = l
+        self.__gen = gen
+        self.name = name
+
+    def __iter__(self):
+        return self
+
+    def next(self):
+        with self.lock:
+            return self.__gen.next()
+
+    @staticmethod
+    def wrapper(method, index_name, meth_name, l=None):
+        @wraps(method)
+        def _inner(*args, **kwargs):
+            res = method(*args, **kwargs)
+            return th_safe_gen(index_name + "_" + meth_name, res, l)
+        return _inner
+
+
+def safe_wrapper(method, lock):
+    @wraps(method)
+    def _inner(*args, **kwargs):
+        with lock:
+            return method(*args, **kwargs)
+    return _inner
+
+
+class SafeDatabase(Database):
+
+    def __init__(self, path, *args, **kwargs):
+        super(SafeDatabase, self).__init__(path, *args, **kwargs)
+        self.indexes_locks = defaultdict(
+            lambda: cdb_environment['rlock_obj']())
+        self.close_open_lock = cdb_environment['rlock_obj']()
+        self.main_lock = cdb_environment['rlock_obj']()
+        self.id_revs = {}
+
+    def __patch_index_gens(self, name):
+        ind = self.indexes_names[name]
+        for c in ('all', 'get_many'):
+            m = getattr(ind, c)
+            if getattr(ind, c + "_orig", None):
+                return
+            m_fixed = th_safe_gen.wrapper(m, name, c, self.indexes_locks[name])
+            setattr(ind, c, m_fixed)
+            setattr(ind, c + '_orig', m)
+
+    def __patch_index_methods(self, name):
+        ind = self.indexes_names[name]
+        lock = self.indexes_locks[name]
+        for curr in dir(ind):
+            meth = getattr(ind, curr)
+            if not curr.startswith('_') and isinstance(meth, MethodType):
+                setattr(ind, curr, safe_wrapper(meth, lock))
+        stor = ind.storage
+        for curr in dir(stor):
+            meth = getattr(stor, curr)
+            if not curr.startswith('_') and isinstance(meth, MethodType):
+                setattr(stor, curr, safe_wrapper(meth, lock))
+
+    def __patch_index(self, name):
+        self.__patch_index_methods(name)
+        self.__patch_index_gens(name)
+
+    def initialize(self, *args, **kwargs):
+        with self.close_open_lock:
+            self.close_open_lock.acquire()
+            res = super(SafeDatabase, self).initialize(*args, **kwargs)
+            for name in self.indexes_names.iterkeys():
+                self.indexes_locks[name] = cdb_environment['rlock_obj']()
+            return res
+
+    def open(self, *args, **kwargs):
+        with self.close_open_lock:
+            res = super(SafeDatabase, self).open(*args, **kwargs)
+            for name in self.indexes_names.iterkeys():
+                self.indexes_locks[name] = cdb_environment['rlock_obj']()
+                self.__patch_index(name)
+            return res
+
+    def create(self, *args, **kwargs):
+        with self.close_open_lock:
+            res = super(SafeDatabase, self).create(*args, **kwargs)
+            for name in self.indexes_names.iterkeys():
+                self.indexes_locks[name] = cdb_environment['rlock_obj']()
+                self.__patch_index(name)
+            return res
+
+    def close(self):
+        with self.close_open_lock:
+            return super(SafeDatabase, self).close()
+
+    def destroy(self):
+        with self.close_open_lock:
+            return super(SafeDatabase, self).destroy()
+
+    def add_index(self, *args, **kwargs):
+        with self.main_lock:
+            res = super(SafeDatabase, self).add_index(*args, **kwargs)
+            if self.opened:
+                self.indexes_locks[res] = cdb_environment['rlock_obj']()
+                self.__patch_index(res)
+            return res
+
+    def _single_update_index(self, index, data, db_data, doc_id):
+        with self.indexes_locks[index.name]:
+            super(SafeDatabase, self)._single_update_index(
+                index, data, db_data, doc_id)
+
+    def _single_delete_index(self, index, data, doc_id, old_data):
+        with self.indexes_locks[index.name]:
+            super(SafeDatabase, self)._single_delete_index(
+                index, data, doc_id, old_data)
+
+    def edit_index(self, *args, **kwargs):
+        with self.main_lock:
+            res = super(SafeDatabase, self).edit_index(*args, **kwargs)
+            if self.opened:
+                self.indexes_locks[res] = cdb_environment['rlock_obj']()
+                self.__patch_index(res)
+            return res
+
+    def set_indexes(self, *args, **kwargs):
+        try:
+            self.main_lock.acquire()
+            super(SafeDatabase, self).set_indexes(*args, **kwargs)
+        finally:
+            self.main_lock.release()
+
+    def reindex_index(self, index, *args, **kwargs):
+        if isinstance(index, basestring):
+            if not index in self.indexes_names:
+                raise PreconditionsException("No index named %s" % index)
+            index = self.indexes_names[index]
+        key = index.name + "reind"
+        self.main_lock.acquire()
+        if key in self.indexes_locks:
+            lock = self.indexes_locks[index.name + "reind"]
+        else:
+            self.indexes_locks[index.name +
+                               "reind"] = cdb_environment['rlock_obj']()
+            lock = self.indexes_locks[index.name + "reind"]
+        self.main_lock.release()
+        try:
+            lock.acquire()
+            super(SafeDatabase, self).reindex_index(
+                index, *args, **kwargs)
+        finally:
+            lock.release()
+
+    def flush(self):
+        try:
+            self.main_lock.acquire()
+            super(SafeDatabase, self).flush()
+        finally:
+            self.main_lock.release()
+
+    def fsync(self):
+        try:
+            self.main_lock.acquire()
+            super(SafeDatabase, self).fsync()
+        finally:
+            self.main_lock.release()
+
+    def _update_id_index(self, _rev, data):
+        with self.indexes_locks['id']:
+            return super(SafeDatabase, self)._update_id_index(_rev, data)
+
+    def _delete_id_index(self, _id, _rev, data):
+        with self.indexes_locks['id']:
+            return super(SafeDatabase, self)._delete_id_index(_id, _rev, data)
+
+    def _update_indexes(self, _rev, data):
+        _id, new_rev, db_data = self._update_id_index(_rev, data)
+        with self.main_lock:
+            self.id_revs[_id] = new_rev
+        for index in self.indexes[1:]:
+            with self.main_lock:
+                curr_rev = self.id_revs.get(_id)  # get last _id, _rev
+                if curr_rev != new_rev:
+                    break  # new update on the way stop current
+            self._single_update_index(index, data, db_data, _id)
+        with self.main_lock:
+            if self.id_revs[_id] == new_rev:
+                del self.id_revs[_id]
+        return _id, new_rev
+
+    def _delete_indexes(self, _id, _rev, data):
+        old_data = self.get('id', _id)
+        if old_data['_rev'] != _rev:
+            raise RevConflict()
+        with self.main_lock:
+            self.id_revs[_id] = _rev
+        for index in self.indexes[1:]:
+            self._single_delete_index(index, data, _id, old_data)
+        self._delete_id_index(_id, _rev, data)
+        with self.main_lock:
+            if self.id_revs[_id] == _rev:
+                del self.id_revs[_id]

libs/CodernityDB/database_super_thread_safe.py

@@ -0,0 +1,112 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright 2011-2013 Codernity (http://codernity.com)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from threading import RLock
+
+from CodernityDB.env import cdb_environment
+
+cdb_environment['mode'] = "threads"
+cdb_environment['rlock_obj'] = RLock
+
+from database import Database
+
+from functools import wraps
+from types import FunctionType, MethodType
+
+from CodernityDB.database_safe_shared import th_safe_gen
+
+
+class SuperLock(type):
+
+    @staticmethod
+    def wrapper(f):
+        @wraps(f)
+        def _inner(*args, **kwargs):
+            db = args[0]
+            with db.super_lock:
+#                print '=>', f.__name__, repr(args[1:])
+                res = f(*args, **kwargs)
+#                if db.opened:
+#                    db.flush()
+#                print '<=', f.__name__, repr(args[1:])
+                return res
+        return _inner
+
+    def __new__(cls, classname, bases, attr):
+        new_attr = {}
+        for base in bases:
+            for b_attr in dir(base):
+                a = getattr(base, b_attr, None)
+                if isinstance(a, MethodType) and not b_attr.startswith('_'):
+                    if b_attr == 'flush' or b_attr == 'flush_indexes':
+                        pass
+                    else:
+                        # setattr(base, b_attr, SuperLock.wrapper(a))
+                        new_attr[b_attr] = SuperLock.wrapper(a)
+        for attr_name, attr_value in attr.iteritems():
+            if isinstance(attr_value, FunctionType) and not attr_name.startswith('_'):
+                attr_value = SuperLock.wrapper(attr_value)
+            new_attr[attr_name] = attr_value
+        new_attr['super_lock'] = RLock()
+        return type.__new__(cls, classname, bases, new_attr)
+
+
+class SuperThreadSafeDatabase(Database):
+    """
+    Thread safe version that always allows single thread to use db.
+    It adds the same lock for all methods, so only one operation can be
+    performed in given time. Completely different implementation
+    than ThreadSafe version (without super word)
+    """
+
+    __metaclass__ = SuperLock
+
+    def __init__(self, *args, **kwargs):
+        super(SuperThreadSafeDatabase, self).__init__(*args, **kwargs)
+
+    def __patch_index_gens(self, name):
+        ind = self.indexes_names[name]
+        for c in ('all', 'get_many'):
+            m = getattr(ind, c)
+            if getattr(ind, c + "_orig", None):
+                return
+            m_fixed = th_safe_gen.wrapper(m, name, c, self.super_lock)
+            setattr(ind, c, m_fixed)
+            setattr(ind, c + '_orig', m)
+
+    def open(self, *args, **kwargs):
+        res = super(SuperThreadSafeDatabase, self).open(*args, **kwargs)
+        for name in self.indexes_names.iterkeys():
+            self.__patch_index_gens(name)
+        return res
+
+    def create(self, *args, **kwargs):
+        res = super(SuperThreadSafeDatabase, self).create(*args, **kwargs)
+        for name in self.indexes_names.iterkeys():
+            self.__patch_index_gens(name)
+            return res
+
+    def add_index(self, *args, **kwargs):
+        res = super(SuperThreadSafeDatabase, self).add_index(*args, **kwargs)
+        self.__patch_index_gens(res)
+        return res
+
+    def edit_index(self, *args, **kwargs):
+        res = super(SuperThreadSafeDatabase, self).edit_index(*args, **kwargs)
+        self.__patch_index_gens(res)
+        return res

libs/CodernityDB/database_thread_safe.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright 2011-2013 Codernity (http://codernity.com)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from threading import RLock
+
+from CodernityDB.env import cdb_environment
+
+cdb_environment['mode'] = "threads"
+cdb_environment['rlock_obj'] = RLock
+
+
+from database_safe_shared import SafeDatabase
+
+
+class ThreadSafeDatabase(SafeDatabase):
+    """
+    Thread safe version of CodernityDB that uses several lock objects,
+    on different methods / different indexes etc. It's completely different
+    implementation of locking than SuperThreadSafe one.
+    """
+    pass

libs/CodernityDB/debug_stuff.py

@@ -0,0 +1,211 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright 2011-2013 Codernity (http://codernity.com)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from CodernityDB.tree_index import TreeBasedIndex
+import struct
+import os
+
+import inspect
+from functools import wraps
+import json
+
+
+class DebugTreeBasedIndex(TreeBasedIndex):
+
+    def __init__(self, *args, **kwargs):
+        super(DebugTreeBasedIndex, self).__init__(*args, **kwargs)
+
+    def print_tree(self):
+        print '-----CURRENT TREE-----'
+        print self.root_flag
+
+        if self.root_flag == 'l':
+            print '---ROOT---'
+            self._print_leaf_data(self.data_start)
+            return
+        else:
+            print '---ROOT---'
+            self._print_node_data(self.data_start)
+            nr_of_el, children_flag = self._read_node_nr_of_elements_and_children_flag(
+                self.data_start)
+            nodes = []
+            for index in range(nr_of_el):
+                l_pointer, key, r_pointer = self._read_single_node_key(
+                    self.data_start, index)
+                nodes.append(l_pointer)
+            nodes.append(r_pointer)
+            print 'ROOT NODES', nodes
+            while children_flag == 'n':
+                self._print_level(nodes, 'n')
+                new_nodes = []
+                for node in nodes:
+                    nr_of_el, children_flag = \
+                        self._read_node_nr_of_elements_and_children_flag(node)
+                    for index in range(nr_of_el):
+                        l_pointer, key, r_pointer = self._read_single_node_key(
+                            node, index)
+                        new_nodes.append(l_pointer)
+                    new_nodes.append(r_pointer)
+                nodes = new_nodes
+            self._print_level(nodes, 'l')
+
+    def _print_level(self, nodes, flag):
+        print '---NEXT LVL---'
+        if flag == 'n':
+            for node in nodes:
+                self._print_node_data(node)
+        elif flag == 'l':
+            for node in nodes:
+                self._print_leaf_data(node)
+
+    def _print_leaf_data(self, leaf_start_position):
+        print 'printing data of leaf at', leaf_start_position
+        nr_of_elements = self._read_leaf_nr_of_elements(leaf_start_position)
+        self.buckets.seek(leaf_start_position)
+        data = self.buckets.read(self.leaf_heading_size +
+                                 nr_of_elements * self.single_leaf_record_size)
+        leaf = struct.unpack('<' + self.leaf_heading_format +
+                             nr_of_elements * self.single_leaf_record_format, data)
+        print leaf
+        print
+
+    def _print_node_data(self, node_start_position):
+        print 'printing data of node at', node_start_position
+        nr_of_elements = self._read_node_nr_of_elements_and_children_flag(
+            node_start_position)[0]
+        self.buckets.seek(node_start_position)
+        data = self.buckets.read(self.node_heading_size + self.pointer_size
+                                 + nr_of_elements * (self.key_size + self.pointer_size))
+        node = struct.unpack('<' + self.node_heading_format + self.pointer_format
+                             + nr_of_elements * (
+                             self.key_format + self.pointer_format),
+                             data)
+        print node
+        print
+# ------------------>
+
+
+def database_step_by_step(db_obj, path=None):
+
+    if not path:
+        # ugly for multiplatform support....
+        p = db_obj.path
+        p1 = os.path.split(p)
+        p2 = os.path.split(p1[0])
+        p3 = '_'.join([p2[1], 'operation_logger.log'])
+        path = os.path.join(os.path.split(p2[0])[0], p3)
+    f_obj = open(path, 'wb')
+
+    __stack = []  # inspect.stack() is not working on pytest etc
+
+    def remove_from_stack(name):
+        for i in range(len(__stack)):
+            if __stack[-i] == name:
+                __stack.pop(-i)
+
+    def __dumper(f):
+        @wraps(f)
+        def __inner(*args, **kwargs):
+            funct_name = f.__name__
+            if funct_name == 'count':
+                name = args[0].__name__
+                meth_args = (name,) + args[1:]
+            elif funct_name in ('reindex_index', 'compact_index'):
+                name = args[0].name
+                meth_args = (name,) + args[1:]
+            else:
+                meth_args = args
+            kwargs_copy = kwargs.copy()
+            res = None
+            __stack.append(funct_name)
+            if funct_name == 'insert':
+                try:
+                    res = f(*args, **kwargs)
+                except:
+                    packed = json.dumps((funct_name,
+                                         meth_args, kwargs_copy, None))
+                    f_obj.write('%s\n' % packed)
+                    f_obj.flush()
+                    raise
+                else:
+                    packed = json.dumps((funct_name,
+                                         meth_args, kwargs_copy, res))
+                f_obj.write('%s\n' % packed)
+                f_obj.flush()
+            else:
+                if funct_name == 'get':
+                    for curr in __stack:
+                        if ('delete' in curr or 'update' in curr) and not curr.startswith('test'):
+                            remove_from_stack(funct_name)
+                            return f(*args, **kwargs)
+                packed = json.dumps((funct_name, meth_args, kwargs_copy))
+                f_obj.write('%s\n' % packed)
+                f_obj.flush()
+                res = f(*args, **kwargs)
+            remove_from_stack(funct_name)
+            return res
+        return __inner
+
+    for meth_name, meth_f in inspect.getmembers(db_obj, predicate=inspect.ismethod):
+        if not meth_name.startswith('_'):
+            setattr(db_obj, meth_name, __dumper(meth_f))
+
+    setattr(db_obj, 'operation_logger', f_obj)
+
+
+def database_from_steps(db_obj, path):
+    # db_obj.insert=lambda data : insert_for_debug(db_obj, data)
+    with open(path, 'rb') as f_obj:
+        for current in f_obj:
+            line = json.loads(current[:-1])
+            if line[0] == 'count':
+                obj = getattr(db_obj, line[1][0])
+                line[1] = [obj] + line[1][1:]
+            name = line[0]
+            if name == 'insert':
+                try:
+                    line[1][0].pop('_rev')
+                except:
+                    pass
+            elif name in ('delete', 'update'):
+                el = db_obj.get('id', line[1][0]['_id'])
+                line[1][0]['_rev'] = el['_rev']
+#                print 'FROM STEPS doing', line
+            meth = getattr(db_obj, line[0], None)
+            if not meth:
+                raise Exception("Method = `%s` not found" % line[0])
+
+            meth(*line[1], **line[2])
+
+
+# def insert_for_debug(self, data):
+#
+#    _rev = data['_rev']
+#
+#    if not '_id' in data:
+#        _id = uuid4().hex
+#    else:
+#        _id = data['_id']
+#    data['_id'] = _id
+#    try:
+#        _id = bytes(_id)
+#    except:
+#        raise DatabaseException("`_id` must be valid bytes object")
+#    self._insert_indexes(_id, _rev, data)
+#    ret = {'_id': _id, '_rev': _rev}
+#    data.update(ret)
+#    return ret

libs/CodernityDB/env.py

@@ -0,0 +1,25 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright 2011-2013 Codernity (http://codernity.com)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+It's CodernityDB environment.
+Handles internal informations.'
+"""
+
+cdb_environment = {
+    'mode': 'normal'
+}

libs/CodernityDB/hash_index.py

@@ -0,0 +1,880 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright 2011-2013 Codernity (http://codernity.com)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from CodernityDB.index import (Index,
+                               IndexException,
+                               DocIdNotFound,
+                               ElemNotFound,
+                               TryReindexException,
+                               IndexPreconditionsException)
+
+import os
+import marshal
+import io
+import struct
+import shutil
+
+from CodernityDB.storage import IU_Storage, DummyStorage
+
+from CodernityDB.env import cdb_environment
+
+if cdb_environment.get('rlock_obj'):
+    from CodernityDB import patch
+    patch.patch_cache_rr(cdb_environment['rlock_obj'])
+
+from CodernityDB.rr_cache import cache1lvl
+
+
+from CodernityDB.misc import random_hex_32
+
+try:
+    from CodernityDB import __version__
+except ImportError:
+    from __init__ import __version__
+
+
+class IU_HashIndex(Index):
+    """
+    That class is for Internal Use only, if you want to use HashIndex just subclass the :py:class:`HashIndex` instead this one.
+
+    That design is because main index logic should be always in database not in custom user indexes.
+    """
+
+    def __init__(self, db_path, name, entry_line_format='<32s{key}IIcI', hash_lim=0xfffff, storage_class=None, key_format='c'):
+        """
+        The index is capable to solve conflicts by `Separate chaining`
+        :param db_path: database path
+        :type db_path: string
+        :param name: index name
+        :type name: ascii string
+        :param line_format: line format, `key_format` parameter value will replace `{key}` if present.
+        :type line_format: string (32s{key}IIcI by default) {doc_id}{hash_key}{start}{size}{status}{next}
+        :param hash_lim: maximum hash functon results (remember about birthday problem) count from 0
+        :type hash_lim: integer
+        :param storage_class: Storage class by default it will open standard :py:class:`CodernityDB.storage.Storage` (if string has to be accesible by globals()[storage_class])
+        :type storage_class: class name which will be instance of CodernityDB.storage.Storage instance or None
+        :param key_format: a index key format
+        """
+        if key_format and '{key}' in entry_line_format:
+            entry_line_format = entry_line_format.replace('{key}', key_format)
+        super(IU_HashIndex, self).__init__(db_path, name)
+        self.hash_lim = hash_lim
+        if not storage_class:
+            storage_class = IU_Storage
+        if storage_class and not isinstance(storage_class, basestring):
+            storage_class = storage_class.__name__
+        self.storage_class = storage_class
+        self.storage = None
+
+        self.bucket_line_format = "<I"
+        self.bucket_line_size = struct.calcsize(self.bucket_line_format)
+        self.entry_line_format = entry_line_format
+        self.entry_line_size = struct.calcsize(self.entry_line_format)
+
+        cache = cache1lvl(100)
+        self._find_key = cache(self._find_key)
+        self._locate_doc_id = cache(self._locate_doc_id)
+        self.bucket_struct = struct.Struct(self.bucket_line_format)
+        self.entry_struct = struct.Struct(self.entry_line_format)
+        self.data_start = (
+            self.hash_lim + 1) * self.bucket_line_size + self._start_ind + 2
+
+    def _fix_params(self):
+        super(IU_HashIndex, self)._fix_params()
+        self.bucket_line_size = struct.calcsize(self.bucket_line_format)
+        self.entry_line_size = struct.calcsize(self.entry_line_format)
+        self.bucket_struct = struct.Struct(self.bucket_line_format)
+        self.entry_struct = struct.Struct(self.entry_line_format)
+        self.data_start = (
+            self.hash_lim + 1) * self.bucket_line_size + self._start_ind + 2
+
+    def open_index(self):
+        if not os.path.isfile(os.path.join(self.db_path, self.name + '_buck')):
+            raise IndexException("Doesn't exists")
+        self.buckets = io.open(
+            os.path.join(self.db_path, self.name + "_buck"), 'r+b', buffering=0)
+        self._fix_params()
+        self._open_storage()
+
+    def create_index(self):
+        if os.path.isfile(os.path.join(self.db_path, self.name + '_buck')):
+            raise IndexException('Already exists')
+        with io.open(os.path.join(self.db_path, self.name + "_buck"), 'w+b') as f:
+            props = dict(name=self.name,
+                         bucket_line_format=self.bucket_line_format,
+                         entry_line_format=self.entry_line_format,
+                         hash_lim=self.hash_lim,
+                         version=self.__version__,
+                         storage_class=self.storage_class)
+            f.write(marshal.dumps(props))
+        self.buckets = io.open(
+            os.path.join(self.db_path, self.name + "_buck"), 'r+b', buffering=0)
+        self._create_storage()
+
+    def destroy(self):
+        super(IU_HashIndex, self).destroy()
+        self._clear_cache()
+
+    def _open_storage(self):
+        s = globals()[self.storage_class]
+        if not self.storage:
+            self.storage = s(self.db_path, self.name)
+        self.storage.open()
+
+    def _create_storage(self):
+        s = globals()[self.storage_class]
+        if not self.storage:
+            self.storage = s(self.db_path, self.name)
+        self.storage.create()
+
+    # def close_index(self):
+    #     self.buckets.flush()
+    #     self.buckets.close()
+    #     self.storage.close()
+#    @lfu_cache(100)
+    def _find_key(self, key):
+        """
+        Find the key position
+
+        :param key: the key to find
+        """
+        start_position = self._calculate_position(key)
+        self.buckets.seek(start_position)
+        curr_data = self.buckets.read(self.bucket_line_size)
+        if curr_data:
+            location = self.bucket_struct.unpack(curr_data)[0]
+            if not location:
+                return None, None, 0, 0, 'u'
+            found_at, doc_id, l_key, start, size, status, _next = self._locate_key(
+                key, location)
+            if status == 'd':  # when first record from many is deleted
+                while True:
+                    found_at, doc_id, l_key, start, size, status, _next = self._locate_key(
+                        key, _next)
+                    if status != 'd':
+                        break
+            return doc_id, l_key, start, size, status
+        else:
+            return None, None, 0, 0, 'u'
+
+    def _find_key_many(self, key, limit=1, offset=0):
+        location = None
+        start_position = self._calculate_position(key)
+        self.buckets.seek(start_position)
+        curr_data = self.buckets.read(self.bucket_line_size)
+        if curr_data:
+            location = self.bucket_struct.unpack(curr_data)[0]
+        while offset:
+            if not location:
+                break
+            try:
+                found_at, doc_id, l_key, start, size, status, _next = self._locate_key(
+                    key, location)
+            except IndexException:
+                break
+            else:
+                if status != 'd':
+                    if l_key == key:  # in case of hash function conflicts
+                        offset -= 1
+                location = _next
+        while limit:
+            if not location:
+                break
+            try:
+                found_at, doc_id, l_key, start, size, status, _next = self._locate_key(
+                    key, location)
+            except IndexException:
+                break
+            else:
+                if status != 'd':
+                    if l_key == key:  # in case of hash function conflicts
+                        yield doc_id, start, size, status
+                        limit -= 1
+                location = _next
+
+    def _calculate_position(self, key):
+        return abs(hash(key) & self.hash_lim) * self.bucket_line_size + self._start_ind
+
+    # TODO add cache!
+    def _locate_key(self, key, start):
+        """
+        Locate position of the key, it will iterate using `next` field in record
+        until required key will be find.
+
+        :param key: the key to locate
+        :param start: position to start from
+        """
+        location = start
+        while True:
+            self.buckets.seek(location)
+            data = self.buckets.read(self.entry_line_size)
+            # todo, maybe partial read there...
+            try:
+                doc_id, l_key, start, size, status, _next = self.entry_struct.unpack(data)
+            except struct.error:
+                raise ElemNotFound(
+                    "Not found")  # not found but might be also broken
+            if l_key == key:
+                break
+            else:
+                if not _next:
+                    # not found
+                    raise ElemNotFound("Not found")
+                else:
+                    location = _next  # go to next record
+        return location, doc_id, l_key, start, size, status, _next
+
+#    @lfu_cache(100)
+    def _locate_doc_id(self, doc_id, key, start):
+        """
+        Locate position of the doc_id, it will iterate using `next` field in record
+        until required key will be find.
+
+        :param doc_id: the doc_id to locate
+        :param key: key value
+        :param start: position to start from
+        """
+        location = start
+        while True:
+            self.buckets.seek(location)
+            data = self.buckets.read(self.entry_line_size)
+            try:
+                l_doc_id, l_key, start, size, status, _next = self.entry_struct.unpack(data)
+            except:
+                raise DocIdNotFound(
+                    "Doc_id '%s' for '%s' not found" % (doc_id, key))
+            if l_doc_id == doc_id and l_key == key:  # added for consistency
+                break
+            else:
+                if not _next:
+                    # not found
+                    raise DocIdNotFound(
+                        "Doc_id '%s' for '%s' not found" % (doc_id, key))
+                else:
+                    location = _next  # go to next record
+        return location, doc_id, l_key, start, size, status, _next
+
+    def _find_place(self, start):
+        """
+        Find a place to where put the key. It will iterate using `next` field in record, until
+        empty `next` found
+
+        :param start: position to start from
+        """
+        location = start
+        while True:
+            self.buckets.seek(location)
+            data = self.buckets.read(self.entry_line_size)
+            # todo, maybe partial read there...
+            doc_id, l_key, start, size, status, _next = self.entry_struct.unpack(data)
+            if not _next or status == 'd':
+                return self.buckets.tell() - self.entry_line_size, doc_id, l_key, start, size, status, _next
+            else:
+                location = _next  # go to next record
+
+    def update(self, doc_id, key, u_start=0, u_size=0, u_status='o'):
+        start_position = self._calculate_position(key)
+        self.buckets.seek(start_position)
+        curr_data = self.buckets.read(self.bucket_line_size)
+        # test if it's unique or not really unique hash
+        if curr_data:
+            location = self.bucket_struct.unpack(curr_data)[0]
+        else:
+            raise ElemNotFound("Location '%s' not found" % doc_id)
+        found_at, _doc_id, _key, start, size, status, _next = self._locate_doc_id(doc_id, key, location)
+        self.buckets.seek(found_at)
+        self.buckets.write(self.entry_struct.pack(doc_id,
+                                                  key,
+                                                  u_start,
+                                                  u_size,
+                                                  u_status,
+                                                  _next))
+        self.flush()
+        self._find_key.delete(key)
+        self._locate_doc_id.delete(doc_id)
+        return True
+
+    def insert(self, doc_id, key, start, size, status='o'):
+        start_position = self._calculate_position(key)
+        self.buckets.seek(start_position)
+        curr_data = self.buckets.read(self.bucket_line_size)
+
+        # conflict occurs?
+        if curr_data:
+            location = self.bucket_struct.unpack(curr_data)[0]
+        else:
+            location = 0
+        if location:
+            # last key with that hash
+            try:
+                found_at, _doc_id, _key, _start, _size, _status, _next = self._locate_doc_id(doc_id, key, location)
+            except DocIdNotFound:
+                found_at, _doc_id, _key, _start, _size, _status, _next = self._find_place(location)
+                self.buckets.seek(0, 2)
+                wrote_at = self.buckets.tell()
+                self.buckets.write(self.entry_struct.pack(doc_id,
+                                                          key,
+                                                          start,
+                                                          size,
+                                                          status,
+                                                          _next))
+#                self.flush()
+                self.buckets.seek(found_at)
+                self.buckets.write(self.entry_struct.pack(_doc_id,
+                                                          _key,
+                                                          _start,
+                                                          _size,
+                                                          _status,
+                                                          wrote_at))
+            else:
+                self.buckets.seek(found_at)
+                self.buckets.write(self.entry_struct.pack(doc_id,
+                                                          key,
+                                                          start,
+                                                          size,
+                                                          status,
+                                                          _next))
+            self.flush()
+            self._locate_doc_id.delete(doc_id)
+            self._find_key.delete(_key)
+            # self._find_key.delete(key)
+            # self._locate_key.delete(_key)
+            return True
+            # raise NotImplementedError
+        else:
+            self.buckets.seek(0, 2)
+            wrote_at = self.buckets.tell()
+
+            # check if position is bigger than all hash entries...
+            if wrote_at < self.data_start:
+                self.buckets.seek(self.data_start)
+                wrote_at = self.buckets.tell()
+
+            self.buckets.write(self.entry_struct.pack(doc_id,
+                                                      key,
+                                                      start,
+                                                      size,
+                                                      status,
+                                                      0))
+#            self.flush()
+            self._find_key.delete(key)
+            self.buckets.seek(start_position)
+            self.buckets.write(self.bucket_struct.pack(wrote_at))
+            self.flush()
+            return True
+
+    def get(self, key):
+        return self._find_key(self.make_key(key))
+
+    def get_many(self, key, limit=1, offset=0):
+        return self._find_key_many(self.make_key(key), limit, offset)
+
+    def all(self, limit=-1, offset=0):
+        self.buckets.seek(self.data_start)
+        while offset:
+            curr_data = self.buckets.read(self.entry_line_size)
+            if not curr_data:
+                break
+            try:
+                doc_id, key, start, size, status, _next = self.entry_struct.unpack(curr_data)
+            except IndexException:
+                break
+            else:
+                if status != 'd':
+                    offset -= 1
+        while limit:
+            curr_data = self.buckets.read(self.entry_line_size)
+            if not curr_data:
+                break
+            try:
+                doc_id, key, start, size, status, _next = self.entry_struct.unpack(curr_data)
+            except IndexException:
+                break
+            else:
+                if status != 'd':
+                    yield doc_id, key, start, size, status
+                    limit -= 1
+
+    def _fix_link(self, key, pos_prev, pos_next):
+        # CHECKIT why I need that hack
+        if pos_prev >= self.data_start:
+            self.buckets.seek(pos_prev)
+            data = self.buckets.read(self.entry_line_size)
+            if data:
+                doc_id, l_key, start, size, status, _next = self.entry_struct.unpack(data)
+                self.buckets.seek(pos_prev)
+                self.buckets.write(self.entry_struct.pack(doc_id,
+                                                          l_key,
+                                                          start,
+                                                          size,
+                                                          status,
+                                                          pos_next))
+                self.flush()
+        if pos_next:
+            self.buckets.seek(pos_next)
+            data = self.buckets.read(self.entry_line_size)
+            if data:
+                doc_id, l_key, start, size, status, _next = self.entry_struct.unpack(data)
+                self.buckets.seek(pos_next)
+                self.buckets.write(self.entry_struct.pack(doc_id,
+                                                          l_key,
+                                                          start,
+                                                          size,
+                                                          status,
+                                                          _next))
+                self.flush()
+        return
+
+    def delete(self, doc_id, key, start=0, size=0):
+        start_position = self._calculate_position(key)
+        self.buckets.seek(start_position)
+        curr_data = self.buckets.read(self.bucket_line_size)
+        if curr_data:
+            location = self.bucket_struct.unpack(curr_data)[0]
+        else:
+            # case happens when trying to delete element with new index key in data
+            # after adding new index to database without reindex
+            raise TryReindexException()
+        found_at, _doc_id, _key, start, size, status, _next = self._locate_doc_id(doc_id, key, location)
+        self.buckets.seek(found_at)
+        self.buckets.write(self.entry_struct.pack(doc_id,
+                                                  key,
+                                                  start,
+                                                  size,
+                                                  'd',
+                                                  _next))
+        self.flush()
+        # self._fix_link(_key, _prev, _next)
+        self._find_key.delete(key)
+        self._locate_doc_id.delete(doc_id)
+        return True
+
+    def compact(self, hash_lim=None):
+
+        if not hash_lim:
+            hash_lim = self.hash_lim
+
+        compact_ind = self.__class__(
+            self.db_path, self.name + '_compact', hash_lim=hash_lim)
+        compact_ind.create_index()
+
+        gen = self.all()
+        while True:
+            try:
+                doc_id, key, start, size, status = gen.next()
+            except StopIteration:
+                break
+            self.storage._f.seek(start)
+            value = self.storage._f.read(size)
+            start_ = compact_ind.storage._f.tell()
+            compact_ind.storage._f.write(value)
+            compact_ind.insert(doc_id, key, start_, size, status)
+
+        compact_ind.close_index()
+        original_name = self.name
+        # os.unlink(os.path.join(self.db_path, self.name + "_buck"))
+        self.close_index()
+        shutil.move(os.path.join(compact_ind.db_path, compact_ind.
+                                 name + "_buck"), os.path.join(self.db_path, self.name + "_buck"))
+        shutil.move(os.path.join(compact_ind.db_path, compact_ind.
+                                 name + "_stor"), os.path.join(self.db_path, self.name + "_stor"))
+        # self.name = original_name
+        self.open_index()  # reload...
+        self.name = original_name
+        self._save_params(dict(name=original_name))
+        self._fix_params()
+        self._clear_cache()
+        return True
+
+    def make_key(self, key):
+        return key
+
+    def make_key_value(self, data):
+        return '1', data
+
+    def _clear_cache(self):
+        self._find_key.clear()
+        self._locate_doc_id.clear()
+
+    def close_index(self):
+        super(IU_HashIndex, self).close_index()
+        self._clear_cache()
+
+
+class IU_UniqueHashIndex(IU_HashIndex):
+    """
+    Index for *unique* keys! Designed to be a **id** index.
+
+    That class is for Internal Use only, if you want to use UniqueHashIndex just subclass the :py:class:`UniqueHashIndex` instead this one.
+
+    That design is because main index logic should be always in database not in custom user indexes.
+    """
+
+    def __init__(self, db_path, name, entry_line_format="<32s8sIIcI", *args, **kwargs):
+        if 'key' in kwargs:
+            raise IndexPreconditionsException(
+                "UniqueHashIndex doesn't accept key parameter'")
+        super(IU_UniqueHashIndex, self).__init__(db_path, name,
+                                                 entry_line_format, *args, **kwargs)
+        self.create_key = random_hex_32  # : set the function to create random key when no _id given
+        # self.entry_struct=struct.Struct(entry_line_format)
+
+#    @lfu_cache(100)
+    def _find_key(self, key):
+        """
+        Find the key position
+
+        :param key: the key to find
+        """
+        start_position = self._calculate_position(key)
+        self.buckets.seek(start_position)
+        curr_data = self.buckets.read(self.bucket_line_size)
+        if curr_data:
+            location = self.bucket_struct.unpack(curr_data)[0]
+            found_at, l_key, rev, start, size, status, _next = self._locate_key(
+                key, location)
+            return l_key, rev, start, size, status
+        else:
+            return None, None, 0, 0, 'u'
+
+    def _find_key_many(self, *args, **kwargs):
+        raise NotImplementedError()
+
+    def _find_place(self, start, key):
+        """
+        Find a place to where put the key. It will iterate using `next` field in record, until
+        empty `next` found
+
+        :param start: position to start from
+        """
+        location = start
+        while True:
+            self.buckets.seek(location)
+            data = self.buckets.read(self.entry_line_size)
+            # todo, maybe partial read there...
+            l_key, rev, start, size, status, _next = self.entry_struct.unpack(
+                data)
+            if l_key == key:
+                raise IndexException("The '%s' key already exists" % key)
+            if not _next or status == 'd':
+                return self.buckets.tell() - self.entry_line_size, l_key, rev, start, size, status, _next
+            else:
+                location = _next  # go to next record
+
+    # @lfu_cache(100)
+    def _locate_key(self, key, start):
+        """
+        Locate position of the key, it will iterate using `next` field in record
+        until required key will be find.
+
+        :param key: the key to locate
+        :param start: position to start from
+        """
+        location = start
+        while True:
+            self.buckets.seek(location)
+            data = self.buckets.read(self.entry_line_size)
+            # todo, maybe partial read there...
+            try:
+                l_key, rev, start, size, status, _next = self.entry_struct.unpack(data)
+            except struct.error:
+                raise ElemNotFound("Location '%s' not found" % key)
+            if l_key == key:
+                break
+            else:
+                if not _next:
+                    # not found
+                    raise ElemNotFound("Location '%s' not found" % key)
+                else:
+                    location = _next  # go to next record
+        return self.buckets.tell() - self.entry_line_size, l_key, rev, start, size, status, _next
+
+    def update(self, key, rev, u_start=0, u_size=0, u_status='o'):
+        start_position = self._calculate_position(key)
+        self.buckets.seek(start_position)
+        curr_data = self.buckets.read(self.bucket_line_size)
+        # test if it's unique or not really unique hash
+
+        if curr_data:
+            location = self.bucket_struct.unpack(curr_data)[0]
+        else:
+            raise ElemNotFound("Location '%s' not found" % key)
+        found_at, _key, _rev, start, size, status, _next = self._locate_key(
+            key, location)
+        if u_start == 0:
+            u_start = start
+        if u_size == 0:
+            u_size = size
+        self.buckets.seek(found_at)
+        self.buckets.write(self.entry_struct.pack(key,
+                                                  rev,
+                                                  u_start,
+                                                  u_size,
+                                                  u_status,
+                                                  _next))
+        self.flush()
+        self._find_key.delete(key)
+        return True
+
+    def insert(self, key, rev, start, size, status='o'):
+        start_position = self._calculate_position(key)
+        self.buckets.seek(start_position)
+        curr_data = self.buckets.read(self.bucket_line_size)
+
+        # conflict occurs?
+        if curr_data:
+            location = self.bucket_struct.unpack(curr_data)[0]
+        else:
+            location = 0
+        if location:
+            # last key with that hash
+            found_at, _key, _rev, _start, _size, _status, _next = self._find_place(
+                location, key)
+            self.buckets.seek(0, 2)
+            wrote_at = self.buckets.tell()
+
+            # check if position is bigger than all hash entries...
+            if wrote_at < self.data_start:
+                self.buckets.seek(self.data_start)
+                wrote_at = self.buckets.tell()
+
+            self.buckets.write(self.entry_struct.pack(key,
+                                                      rev,
+                                                      start,
+                                                      size,
+                                                      status,
+                                                      _next))
+
+#            self.flush()
+            self.buckets.seek(found_at)
+            self.buckets.write(self.entry_struct.pack(_key,
+                                                      _rev,
+                                                      _start,
+                                                      _size,
+                                                      _status,
+                                                      wrote_at))
+            self.flush()
+            self._find_key.delete(_key)
+            # self._locate_key.delete(_key)
+            return True
+            # raise NotImplementedError
+        else:
+            self.buckets.seek(0, 2)
+            wrote_at = self.buckets.tell()
+
+            # check if position is bigger than all hash entries...
+            if wrote_at < self.data_start:
+                self.buckets.seek(self.data_start)
+                wrote_at = self.buckets.tell()
+
+            self.buckets.write(self.entry_struct.pack(key,
+                                                      rev,
+                                                      start,
+                                                      size,
+                                                      status,
+                                                      0))
+#            self.flush()
+            self.buckets.seek(start_position)
+            self.buckets.write(self.bucket_struct.pack(wrote_at))
+            self.flush()
+            self._find_key.delete(key)
+            return True
+
+    def all(self, limit=-1, offset=0):
+        self.buckets.seek(self.data_start)
+        while offset:
+            curr_data = self.buckets.read(self.entry_line_size)
+            if not curr_data:
+                break
+            try:
+                doc_id, rev, start, size, status, next = self.entry_struct.unpack(curr_data)
+            except IndexException:
+                break
+            else:
+                if status != 'd':
+                    offset -= 1
+
+        while limit:
+            curr_data = self.buckets.read(self.entry_line_size)
+            if not curr_data:
+                break
+            try:
+                doc_id, rev, start, size, status, next = self.entry_struct.unpack(curr_data)
+            except IndexException:
+                break
+            else:
+                if status != 'd':
+                    yield doc_id, rev, start, size, status
+                    limit -= 1
+
+    def get_many(self, *args, **kwargs):
+        raise NotImplementedError()
+
+    def delete(self, key, start=0, size=0):
+        self.update(key, '00000000', start, size, 'd')
+
+    def make_key_value(self, data):
+        _id = data['_id']
+        try:
+            _id = bytes(data['_id'])
+        except:
+            raise IndexPreconditionsException(
+                "_id must be valid string/bytes object")
+        if len(_id) != 32:
+            raise IndexPreconditionsException("Invalid _id lenght")
+        del data['_id']
+        del data['_rev']
+        return _id, data
+
+    def destroy(self):
+        Index.destroy(self)
+        self._clear_cache()
+
+    def _clear_cache(self):
+        self._find_key.clear()
+
+    def insert_with_storage(self, _id, _rev, value):
+        if value:
+            start, size = self.storage.insert(value)
+        else:
+            start = 1
+            size = 0
+        return self.insert(_id, _rev, start, size)
+
+    def update_with_storage(self, _id, _rev, value):
+        if value:
+            start, size = self.storage.insert(value)
+        else:
+            start = 1
+            size = 0
+        return self.update(_id, _rev, start, size)
+
+
+class DummyHashIndex(IU_HashIndex):
+    def __init__(self, db_path, name, entry_line_format="<32s4sIIcI", *args, **kwargs):
+        super(DummyHashIndex, self).__init__(db_path, name,
+                                             entry_line_format, *args, **kwargs)
+        self.create_key = random_hex_32  # : set the function to create random key when no _id given
+        # self.entry_struct=struct.Struct(entry_line_format)
+
+    def update(self, *args, **kwargs):
+        return True
+
+    def insert(self, *args, **kwargs):
+        return True
+
+    def all(self, *args, **kwargs):
+        raise StopIteration
+
+    def get(self, *args, **kwargs):
+        raise ElemNotFound
+
+    def get_many(self, *args, **kwargs):
+        raise StopIteration
+
+    def delete(self, *args, **kwargs):
+        pass
+
+    def make_key_value(self, data):
+        return '1', {'_': 1}
+
+    def destroy(self):
+        pass
+
+    def _clear_cache(self):
+        pass
+
+    def _open_storage(self):
+        if not self.storage:
+            self.storage = DummyStorage()
+        self.storage.open()
+
+    def _create_storage(self):
+        if not self.storage:
+            self.storage = DummyStorage()
+        self.storage.create()
+
+
+class IU_MultiHashIndex(IU_HashIndex):
+    """
+    Class that allows to index more than one key per database record.
+
+    It operates very well on GET/INSERT. It's not optimized for
+    UPDATE operations (will always readd everything)
+    """
+
+    def __init__(self, *args, **kwargs):
+        super(IU_MultiHashIndex, self).__init__(*args, **kwargs)
+
+    def insert(self, doc_id, key, start, size, status='o'):
+        if isinstance(key, (list, tuple)):
+            key = set(key)
+        elif not isinstance(key, set):
+            key = set([key])
+        ins = super(IU_MultiHashIndex, self).insert
+        for curr_key in key:
+            ins(doc_id, curr_key, start, size, status)
+        return True
+
+    def update(self, doc_id, key, u_start, u_size, u_status='o'):
+        if isinstance(key, (list, tuple)):
+            key = set(key)
+        elif not isinstance(key, set):
+            key = set([key])
+        upd = super(IU_MultiHashIndex, self).update
+        for curr_key in key:
+            upd(doc_id, curr_key, u_start, u_size, u_status)
+
+    def delete(self, doc_id, key, start=0, size=0):
+        if isinstance(key, (list, tuple)):
+            key = set(key)
+        elif not isinstance(key, set):
+            key = set([key])
+        delete = super(IU_MultiHashIndex, self).delete
+        for curr_key in key:
+            delete(doc_id, curr_key, start, size)
+
+    def get(self, key):
+        return super(IU_MultiHashIndex, self).get(key)
+
+    def make_key_value(self, data):
+        raise NotImplementedError()
+
+
+# classes for public use, done in this way because of
+# generation static files with indexes (_index directory)
+
+
+class HashIndex(IU_HashIndex):
+    """
+    That class is designed to be used in custom indexes.
+    """
+    pass
+
+
+class UniqueHashIndex(IU_UniqueHashIndex):
+    """
+    That class is designed to be used in custom indexes. It's designed to be **id** index.
+    """
+    pass
+
+
+class MultiHashIndex(IU_MultiHashIndex):
+    """
+    That class is designed to be used in custom indexes.
+    """

libs/CodernityDB/index.py

@@ -0,0 +1,195 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright 2011-2013 Codernity (http://codernity.com)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import os
+import marshal
+
+import struct
+import shutil
+
+from CodernityDB.storage import IU_Storage, DummyStorage
+
+try:
+    from CodernityDB import __version__
+except ImportError:
+    from __init__ import __version__
+
+
+import io
+
+
+class IndexException(Exception):
+    pass
+
+
+class IndexNotFoundException(IndexException):
+    pass
+
+
+class ReindexException(IndexException):
+    pass
+
+
+class TryReindexException(ReindexException):
+    pass
+
+
+class ElemNotFound(IndexException):
+    pass
+
+
+class DocIdNotFound(ElemNotFound):
+    pass
+
+
+class IndexConflict(IndexException):
+    pass
+
+
+class IndexPreconditionsException(IndexException):
+    pass
+
+
+class Index(object):
+
+    __version__ = __version__
+
+    custom_header = ""  # : use it for imports required by your index
+
+    def __init__(self,
+                 db_path,
+                 name):
+        self.name = name
+        self._start_ind = 500
+        self.db_path = db_path
+
+    def open_index(self):
+        if not os.path.isfile(os.path.join(self.db_path, self.name + '_buck')):
+            raise IndexException("Doesn't exists")
+        self.buckets = io.open(
+            os.path.join(self.db_path, self.name + "_buck"), 'r+b', buffering=0)
+        self._fix_params()
+        self._open_storage()
+
+    def _close(self):
+        self.buckets.close()
+        self.storage.close()
+
+    def close_index(self):
+        self.flush()
+        self.fsync()
+        self._close()
+
+    def create_index(self):
+        raise NotImplementedError()
+
+    def _fix_params(self):
+        self.buckets.seek(0)
+        props = marshal.loads(self.buckets.read(self._start_ind))
+        for k, v in props.iteritems():
+            self.__dict__[k] = v
+        self.buckets.seek(0, 2)
+
+    def _save_params(self, in_params={}):
+        self.buckets.seek(0)
+        props = marshal.loads(self.buckets.read(self._start_ind))
+        props.update(in_params)
+        self.buckets.seek(0)
+        data = marshal.dumps(props)
+        if len(data) > self._start_ind:
+            raise IndexException("To big props")
+        self.buckets.write(data)
+        self.flush()
+        self.buckets.seek(0, 2)
+        self.__dict__.update(props)
+
+    def _open_storage(self, *args, **kwargs):
+        pass
+
+    def _create_storage(self, *args, **kwargs):
+        pass
+
+    def _destroy_storage(self, *args, **kwargs):
+        self.storage.destroy()
+
+    def _find_key(self, key):
+        raise NotImplementedError()
+
+    def update(self, doc_id, key, start, size):
+        raise NotImplementedError()
+
+    def insert(self, doc_id, key, start, size):
+        raise NotImplementedError()
+
+    def get(self, key):
+        raise NotImplementedError()
+
+    def get_many(self, key, start_from=None, limit=0):
+        raise NotImplementedError()
+
+    def all(self, start_pos):
+        raise NotImplementedError()
+
+    def delete(self, key, start, size):
+        raise NotImplementedError()
+
+    def make_key_value(self, data):
+        raise NotImplementedError()
+
+    def make_key(self, data):
+        raise NotImplementedError()
+
+    def compact(self, *args, **kwargs):
+        raise NotImplementedError()
+
+    def destroy(self, *args, **kwargs):
+        self._close()
+        bucket_file = os.path.join(self.db_path, self.name + '_buck')
+        os.unlink(bucket_file)
+        self._destroy_storage()
+        self._find_key.clear()
+
+    def flush(self):
+        try:
+            self.buckets.flush()
+            self.storage.flush()
+        except:
+            pass
+
+    def fsync(self):
+        try:
+            os.fsync(self.buckets.fileno())
+            self.storage.fsync()
+        except:
+            pass
+
+    def update_with_storage(self, doc_id, key, value):
+        if value:
+            start, size = self.storage.insert(value)
+        else:
+            start = 1
+            size = 0
+        return self.update(doc_id, key, start, size)
+
+    def insert_with_storage(self, doc_id, key, value):
+        if value:
+            start, size = self.storage.insert(value)
+        else:
+            start = 1
+            size = 0
+        return self.insert(doc_id, key, start, size)

libs/CodernityDB/indexcreator.py

@@ -0,0 +1,645 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright 2011-2013 Codernity (http://codernity.com)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import re
+import tokenize
+import token
+import uuid
+
+
+class IndexCreatorException(Exception):
+    def __init__(self, ex, line=None):
+        self.ex = ex
+        self.line = line
+
+    def __str__(self):
+        if self.line:
+            return repr(self.ex + "(in line: %d)" % self.line)
+        return repr(self.ex)
+
+
+class IndexCreatorFunctionException(IndexCreatorException):
+    pass
+
+
+class IndexCreatorValueException(IndexCreatorException):
+    pass
+
+
+class Parser(object):
+    def __init__(self):
+        pass
+
+    def parse(self, data, name=None):
+        if not name:
+            self.name = "_" + uuid.uuid4().hex
+        else:
+            self.name = name
+
+        self.ind = 0
+        self.stage = 0
+        self.logic = ['and', 'or', 'in']
+        self.logic2 = ['&', '|']
+        self.allowed_props = {'TreeBasedIndex': ['type', 'name', 'key_format', 'node_capacity', 'pointer_format', 'meta_format'],
+                              'HashIndex': ['type', 'name', 'key_format', 'hash_lim', 'entry_line_format'],
+                              'MultiHashIndex': ['type', 'name', 'key_format', 'hash_lim', 'entry_line_format'],
+                              'MultiTreeBasedIndex': ['type', 'name', 'key_format', 'node_capacity', 'pointer_format', 'meta_format']
+                              }
+        self.funcs = {'md5': (['md5'], ['.digest()']),
+                      'len': (['len'], []),
+                      'str': (['str'], []),
+                      'fix_r': (['self.fix_r'], []),
+                      'prefix': (['self.prefix'], []),
+                      'infix': (['self.infix'], []),
+                      'suffix': (['self.suffix'], [])
+                      }
+        self.handle_int_imports = {'infix': "from itertools import izip\n"}
+
+        self.funcs_with_body = {'fix_r':
+                                ("""    def fix_r(self,s,l):
+        e = len(s)
+        if e == l:
+            return s
+        elif e > l:
+            return s[:l]
+        else:
+            return s.rjust(l,'_')\n""", False),
+                                'prefix':
+                                ("""    def prefix(self,s,m,l,f):
+        t = len(s)
+        if m < 1:
+            m = 1
+        o = set()
+        if t > l:
+            s = s[:l]
+            t = l
+        while m <= t:
+            o.add(s.rjust(f,'_'))
+            s = s[:-1]
+            t -= 1
+        return o\n""", False),
+                                'suffix':
+                                ("""    def suffix(self,s,m,l,f):
+        t = len(s)
+        if m < 1:
+            m = 1
+        o = set()
+        if t > l:
+            s = s[t-l:]
+            t = len(s)
+        while m <= t:
+            o.add(s.rjust(f,'_'))
+            s = s[1:]
+            t -= 1
+        return o\n""", False),
+                                'infix':
+                                ("""    def infix(self,s,m,l,f):
+        t = len(s)
+        o = set()
+        for x in xrange(m - 1, l):
+            t = (s, )
+            for y in xrange(0, x):
+                t += (s[y + 1:],)
+            o.update(set(''.join(x).rjust(f, '_').lower() for x in izip(*t)))
+        return o\n""", False)}
+        self.none = ['None', 'none', 'null']
+        self.props_assign = ['=', ':']
+        self.all_adj_num_comp = {token.NUMBER: (
+            token.NUMBER, token.NAME, '-', '('),
+            token.NAME: (token.NUMBER, token.NAME, '-', '('),
+            ')': (token.NUMBER, token.NAME, '-', '(')
+        }
+
+        self.all_adj_num_op = {token.NUMBER: (token.NUMBER, token.NAME, '('),
+                               token.NAME: (token.NUMBER, token.NAME, '('),
+                               ')': (token.NUMBER, token.NAME, '(')
+                               }
+        self.allowed_adjacent = {
+            "<=": self.all_adj_num_comp,
+            ">=": self.all_adj_num_comp,
+            ">": self.all_adj_num_comp,
+            "<": self.all_adj_num_comp,
+
+            "==": {token.NUMBER: (token.NUMBER, token.NAME, '('),
+                   token.NAME: (token.NUMBER, token.NAME, token.STRING, '('),
+                   token.STRING: (token.NAME, token.STRING, '('),
+                   ')': (token.NUMBER, token.NAME, token.STRING, '('),
+                   ']': (token.NUMBER, token.NAME, token.STRING, '(')
+                   },
+
+            "+": {token.NUMBER: (token.NUMBER, token.NAME, '('),
+                  token.NAME: (token.NUMBER, token.NAME, token.STRING, '('),
+                  token.STRING: (token.NAME, token.STRING, '('),
+                  ')': (token.NUMBER, token.NAME, token.STRING, '('),
+                  ']': (token.NUMBER, token.NAME, token.STRING, '(')
+                  },
+
+            "-": {token.NUMBER: (token.NUMBER, token.NAME, '('),
+                  token.NAME: (token.NUMBER, token.NAME, '('),
+                  ')': (token.NUMBER, token.NAME, '('),
+                  '<': (token.NUMBER, token.NAME, '('),
+                  '>': (token.NUMBER, token.NAME, '('),
+                  '<=': (token.NUMBER, token.NAME, '('),
+                  '>=': (token.NUMBER, token.NAME, '('),
+                  '==': (token.NUMBER, token.NAME, '('),
+                  ']': (token.NUMBER, token.NAME, '(')
+                  },
+            "*": self.all_adj_num_op,
+            "/": self.all_adj_num_op,
+            "%": self.all_adj_num_op,
+            ",": {token.NUMBER: (token.NUMBER, token.NAME, token.STRING, '{', '[', '('),
+                  token.NAME: (token.NUMBER, token.NAME, token.STRING, '(', '{', '['),
+                  token.STRING: (token.NAME, token.STRING, token.NUMBER, '(', '{', '['),
+                  ')': (token.NUMBER, token.NAME, token.STRING, '(', '{', '['),
+                  ']': (token.NUMBER, token.NAME, token.STRING, '(', '{', '['),
+                  '}': (token.NUMBER, token.NAME, token.STRING, '(', '{', '[')
+                  }
+        }
+
+        def is_num(s):
+            m = re.search('[^0-9*()+\-\s/]+', s)
+            return not m
+
+        def is_string(s):
+            m = re.search('\s*(?P<a>[\'\"]+).*?(?P=a)\s*', s)
+            return m
+        data = re.split('make_key_value\:', data)
+
+        if len(data) < 2:
+            raise IndexCreatorFunctionException(
+                "Couldn't find a definition of make_key_value function!\n")
+
+        spl1 = re.split('make_key\:', data[0])
+        spl2 = re.split('make_key\:', data[1])
+
+        self.funcs_rev = False
+
+        if len(spl1) > 1:
+            data = [spl1[0]] + [data[1]] + [spl1[1]]
+            self.funcs_rev = True
+        elif len(spl2) > 1:
+            data = [data[0]] + spl2
+        else:
+            data.append("key")
+
+        if data[1] == re.search('\s*', data[1], re.S | re.M).group(0):
+            raise IndexCreatorFunctionException("Empty function body ",
+                                                len(re.split('\n', data[0])) + (len(re.split('\n', data[2])) if self.funcs_rev else 1) - 1)
+        if data[2] == re.search('\s*', data[2], re.S | re.M).group(0):
+            raise IndexCreatorFunctionException("Empty function body ",
+                                                len(re.split('\n', data[0])) + (1 if self.funcs_rev else len(re.split('\n', data[1]))) - 1)
+        if data[0] == re.search('\s*', data[0], re.S | re.M).group(0):
+            raise IndexCreatorValueException("You didn't set any properity or you set them not at the begining of the code\n")
+
+        data = [re.split(
+            '\n', data[0]), re.split('\n', data[1]), re.split('\n', data[2])]
+        self.cnt_lines = (len(data[0]), len(data[1]), len(data[2]))
+        ind = 0
+        self.predata = data
+        self.data = [[], [], []]
+        for i, v in enumerate(self.predata[0]):
+            for k, w in enumerate(self.predata[0][i]):
+                if self.predata[0][i][k] in self.props_assign:
+                    if not is_num(self.predata[0][i][k + 1:]) and self.predata[0][i].strip()[:4] != 'type' and self.predata[0][i].strip()[:4] != 'name':
+                        s = self.predata[0][i][k + 1:]
+                        self.predata[0][i] = self.predata[0][i][:k + 1]
+
+                        m = re.search('\s+', s.strip())
+                        if not is_string(s) and not m:
+                            s = "'" + s.strip() + "'"
+                        self.predata[0][i] += s
+                        break
+
+        for n, i in enumerate(self.predata):
+            for k in i:
+                k = k.strip()
+                if k:
+                    self.data[ind].append(k)
+                    self.check_enclosures(k, n)
+            ind += 1
+
+        return self.parse_ex()
+
+    def readline(self, stage):
+        def foo():
+            if len(self.data[stage]) <= self.ind:
+                self.ind = 0
+                return ""
+            else:
+                self.ind += 1
+                return self.data[stage][self.ind - 1]
+        return foo
+
+    def add(self, l, i):
+        def add_aux(*args):
+            # print args,self.ind
+            if len(l[i]) < self.ind:
+                l[i].append([])
+            l[i][self.ind - 1].append(args)
+        return add_aux
+
+    def parse_ex(self):
+        self.index_name = ""
+        self.index_type = ""
+        self.curLine = -1
+        self.con = -1
+        self.brackets = -1
+        self.curFunc = None
+        self.colons = 0
+        self.line_cons = ([], [], [])
+        self.pre_tokens = ([], [], [])
+        self.known_dicts_in_mkv = []
+        self.prop_name = True
+        self.prop_assign = False
+        self.is_one_arg_enough = False
+        self.funcs_stack = []
+        self.last_line = [-1, -1, -1]
+        self.props_set = []
+        self.custom_header = set()
+
+        self.tokens = []
+        self.tokens_head = ['# %s\n' % self.name, 'class %s(' % self.name, '):\n', '    def __init__(self, *args, **kwargs):        ']
+
+        for i in xrange(3):
+            tokenize.tokenize(self.readline(i), self.add(self.pre_tokens, i))
+            # tokenize treats some keyword not in the right way, thats why we
+            # have to change some of them
+            for nk, k in enumerate(self.pre_tokens[i]):
+                for na, a in enumerate(k):
+                    if a[0] == token.NAME and a[1] in self.logic:
+                        self.pre_tokens[i][nk][
+                            na] = (token.OP, a[1], a[2], a[3], a[4])
+
+        for i in self.pre_tokens[1]:
+            self.line_cons[1].append(self.check_colons(i, 1))
+            self.check_adjacents(i, 1)
+            if self.check_for_2nd_arg(i) == -1 and not self.is_one_arg_enough:
+                raise IndexCreatorValueException("No 2nd value to return (did u forget about ',None'?", self.cnt_line_nr(i[0][4], 1))
+            self.is_one_arg_enough = False
+
+        for i in self.pre_tokens[2]:
+            self.line_cons[2].append(self.check_colons(i, 2))
+            self.check_adjacents(i, 2)
+
+        for i in self.pre_tokens[0]:
+            self.handle_prop_line(i)
+
+        self.cur_brackets = 0
+        self.tokens += ['\n        super(%s, self).__init__(*args, **kwargs)\n    def make_key_value(self, data):        ' % self.name]
+
+        for i in self.pre_tokens[1]:
+            for k in i:
+                self.handle_make_value(*k)
+
+        self.curLine = -1
+        self.con = -1
+        self.cur_brackets = 0
+        self.tokens += ['\n    def make_key(self, key):']
+
+        for i in self.pre_tokens[2]:
+            for k in i:
+                self.handle_make_key(*k)
+
+        if self.index_type == "":
+            raise IndexCreatorValueException("Missing index type definition\n")
+        if self.index_name == "":
+            raise IndexCreatorValueException("Missing index name\n")
+
+        self.tokens_head[0] = "# " + self.index_name + "\n" + \
+            self.tokens_head[0]
+
+        for i in self.funcs_with_body:
+            if self.funcs_with_body[i][1]:
+                self.tokens_head.insert(4, self.funcs_with_body[i][0])
+
+        if None in self.custom_header:
+            self.custom_header.remove(None)
+        if self.custom_header:
+            s = '    custom_header = """'
+            for i in self.custom_header:
+                s += i
+            s += '"""\n'
+            self.tokens_head.insert(4, s)
+
+        if self.index_type in self.allowed_props:
+            for i in self.props_set:
+                if i not in self.allowed_props[self.index_type]:
+                    raise IndexCreatorValueException("Properity %s is not allowed for index type: %s" % (i, self.index_type))
+
+        # print "".join(self.tokens_head)
+        # print "----------"
+        # print (" ".join(self.tokens))
+        return "".join(self.custom_header), "".join(self.tokens_head) + (" ".join(self.tokens))
+
+    # has to be run BEFORE tokenize
+    def check_enclosures(self, d, st):
+        encs = []
+        contr = {'(': ')', '{': '}', '[': ']', "'": "'", '"': '"'}
+        ends = [')', '}', ']', "'", '"']
+        for i in d:
+            if len(encs) > 0 and encs[-1] in ['"', "'"]:
+                if encs[-1] == i:
+                    del encs[-1]
+            elif i in contr:
+                encs += [i]
+            elif i in ends:
+                if len(encs) < 1 or contr[encs[-1]] != i:
+                    raise IndexCreatorValueException("Missing opening enclosure for \'%s\'" % i, self.cnt_line_nr(d, st))
+                del encs[-1]
+
+        if len(encs) > 0:
+            raise IndexCreatorValueException("Missing closing enclosure for \'%s\'" % encs[0], self.cnt_line_nr(d, st))
+
+    def check_adjacents(self, d, st):
+        def std_check(d, n):
+            if n == 0:
+                prev = -1
+            else:
+                prev = d[n - 1][1] if d[n - 1][0] == token.OP else d[n - 1][0]
+
+            cur = d[n][1] if d[n][0] == token.OP else d[n][0]
+
+            # there always is an endmarker at the end, but this is a precaution
+            if n + 2 > len(d):
+                nex = -1
+            else:
+                nex = d[n + 1][1] if d[n + 1][0] == token.OP else d[n + 1][0]
+
+            if prev not in self.allowed_adjacent[cur]:
+                raise IndexCreatorValueException("Wrong left value of the %s" % cur, self.cnt_line_nr(line, st))
+
+            # there is an assumption that whole data always ends with 0 marker, the idea prolly needs a rewritting to allow more whitespaces
+            # between tokens, so it will be handled anyway
+            elif nex not in self.allowed_adjacent[cur][prev]:
+                raise IndexCreatorValueException("Wrong right value of the %s" % cur, self.cnt_line_nr(line, st))
+
+        for n, (t, i, _, _, line) in enumerate(d):
+            if t == token.NAME or t == token.STRING:
+                if n + 1 < len(d) and d[n + 1][0] in [token.NAME, token.STRING]:
+                    raise IndexCreatorValueException("Did you forget about an operator in between?", self.cnt_line_nr(line, st))
+            elif i in self.allowed_adjacent:
+                std_check(d, n)
+
+    def check_colons(self, d, st):
+        cnt = 0
+        br = 0
+
+        def check_ret_args_nr(a, s):
+            c_b_cnt = 0
+            s_b_cnt = 0
+            n_b_cnt = 0
+            comas_cnt = 0
+            for _, i, _, _, line in a:
+
+                if c_b_cnt == n_b_cnt == s_b_cnt == 0:
+                    if i == ',':
+                        comas_cnt += 1
+                        if (s == 1 and comas_cnt > 1) or (s == 2 and comas_cnt > 0):
+                            raise IndexCreatorFunctionException("Too much arguments to return", self.cnt_line_nr(line, st))
+                        if s == 0 and comas_cnt > 0:
+                            raise IndexCreatorValueException("A coma here doesn't make any sense", self.cnt_line_nr(line, st))
+
+                    elif i == ':':
+                            if s == 0:
+                                raise IndexCreatorValueException("A colon here doesn't make any sense", self.cnt_line_nr(line, st))
+                            raise IndexCreatorFunctionException("Two colons don't make any sense", self.cnt_line_nr(line, st))
+
+                if i == '{':
+                    c_b_cnt += 1
+                elif i == '}':
+                    c_b_cnt -= 1
+                elif i == '(':
+                    n_b_cnt += 1
+                elif i == ')':
+                    n_b_cnt -= 1
+                elif i == '[':
+                    s_b_cnt += 1
+                elif i == ']':
+                    s_b_cnt -= 1
+
+        def check_if_empty(a):
+            for i in a:
+                if i not in [token.NEWLINE, token.INDENT, token.ENDMARKER]:
+                    return False
+            return True
+        if st == 0:
+            check_ret_args_nr(d, st)
+            return
+
+        for n, i in enumerate(d):
+            if i[1] == ':':
+                if br == 0:
+                    if len(d) < n or check_if_empty(d[n + 1:]):
+                        raise IndexCreatorValueException(
+                            "Empty return value", self.cnt_line_nr(i[4], st))
+                    elif len(d) >= n:
+                        check_ret_args_nr(d[n + 1:], st)
+                    return cnt
+                else:
+                    cnt += 1
+            elif i[1] == '{':
+                br += 1
+            elif i[1] == '}':
+                br -= 1
+        check_ret_args_nr(d, st)
+        return -1
+
+    def check_for_2nd_arg(self, d):
+        c_b_cnt = 0  # curly brackets counter '{}'
+        s_b_cnt = 0  # square brackets counter '[]'
+        n_b_cnt = 0  # normal brackets counter '()'
+
+        def check_2nd_arg(d, ind):
+            d = d[ind[0]:]
+            for t, i, (n, r), _, line in d:
+                if i == '{' or i is None:
+                    return 0
+                elif t == token.NAME:
+                    self.known_dicts_in_mkv.append((i, (n, r)))
+                    return 0
+                elif t == token.STRING or t == token.NUMBER:
+                    raise IndexCreatorValueException("Second return value of make_key_value function has to be a dictionary!", self.cnt_line_nr(line, 1))
+
+        for ind in enumerate(d):
+            t, i, _, _, _ = ind[1]
+            if s_b_cnt == n_b_cnt == c_b_cnt == 0:
+                if i == ',':
+                    return check_2nd_arg(d, ind)
+                elif (t == token.NAME and i not in self.funcs) or i == '{':
+                    self.is_one_arg_enough = True
+
+            if i == '{':
+                c_b_cnt += 1
+                self.is_one_arg_enough = True
+            elif i == '}':
+                c_b_cnt -= 1
+            elif i == '(':
+                n_b_cnt += 1
+            elif i == ')':
+                n_b_cnt -= 1
+            elif i == '[':
+                s_b_cnt += 1
+            elif i == ']':
+                s_b_cnt -= 1
+        return -1
+
+    def cnt_line_nr(self, l, stage):
+        nr = -1
+        for n, i in enumerate(self.predata[stage]):
+            # print i,"|||",i.strip(),"|||",l
+            if l == i.strip():
+                nr = n
+        if nr == -1:
+            return -1
+
+        if stage == 0:
+            return nr + 1
+        elif stage == 1:
+            return nr + self.cnt_lines[0] + (self.cnt_lines[2] - 1 if self.funcs_rev else 0)
+        elif stage == 2:
+            return nr + self.cnt_lines[0] + (self.cnt_lines[1] - 1 if not self.funcs_rev else 0)
+
+        return -1
+
+    def handle_prop_line(self, d):
+        d_len = len(d)
+        if d[d_len - 1][0] == token.ENDMARKER:
+            d_len -= 1
+
+        if d_len < 3:
+            raise IndexCreatorValueException("Can't handle properity assingment ", self.cnt_line_nr(d[0][4], 0))
+
+        if not d[1][1] in self.props_assign:
+            raise IndexCreatorValueException(
+                "Did you forget : or =?", self.cnt_line_nr(d[0][4], 0))
+
+        if d[0][0] == token.NAME or d[0][0] == token.STRING:
+            if d[0][1] in self.props_set:
+                raise IndexCreatorValueException("Properity %s is set more than once" % d[0][1], self.cnt_line_nr(d[0][4], 0))
+            self.props_set += [d[0][1]]
+            if d[0][1] == "type" or d[0][1] == "name":
+                t, tk, _, _, line = d[2]
+
+                if d_len > 3:
+                    raise IndexCreatorValueException(
+                        "Wrong value to assign", self.cnt_line_nr(line, 0))
+
+                if t == token.STRING:
+                    m = re.search('\s*(?P<a>[\'\"]+)(.*?)(?P=a)\s*', tk)
+                    if m:
+                        tk = m.groups()[1]
+                elif t != token.NAME:
+                    raise IndexCreatorValueException(
+                        "Wrong value to assign", self.cnt_line_nr(line, 0))
+
+                if d[0][1] == "type":
+                    if d[2][1] == "TreeBasedIndex":
+                        self.custom_header.add("from CodernityDB.tree_index import TreeBasedIndex\n")
+                    elif d[2][1] == "MultiTreeBasedIndex":
+                        self.custom_header.add("from CodernityDB.tree_index import MultiTreeBasedIndex\n")
+                    elif d[2][1] == "MultiHashIndex":
+                        self.custom_header.add("from CodernityDB.hash_index import MultiHashIndex\n")
+                    self.tokens_head.insert(2, tk)
+                    self.index_type = tk
+                else:
+                    self.index_name = tk
+                return
+            else:
+                self.tokens += ['\n        kwargs["' + d[0][1] + '"]']
+        else:
+            raise IndexCreatorValueException("Can't handle properity assingment ", self.cnt_line_nr(d[0][4], 0))
+
+        self.tokens += ['=']
+
+        self.check_adjacents(d[2:], 0)
+        self.check_colons(d[2:], 0)
+
+        for i in d[2:]:
+            self.tokens += [i[1]]
+
+    def generate_func(self, t, tk, pos_start, pos_end, line, hdata, stage):
+        if self.last_line[stage] != -1 and pos_start[0] > self.last_line[stage] and line != '':
+            raise IndexCreatorFunctionException("This line will never be executed!", self.cnt_line_nr(line, stage))
+        if t == 0:
+            return
+
+        if pos_start[1] == 0:
+            if self.line_cons[stage][pos_start[0] - 1] == -1:
+                self.tokens += ['\n        return']
+                self.last_line[stage] = pos_start[0]
+            else:
+                self.tokens += ['\n        if']
+        elif tk == ':' and self.line_cons[stage][pos_start[0] - 1] > -1:
+            if self.line_cons[stage][pos_start[0] - 1] == 0:
+                self.tokens += [':\n            return']
+                return
+            self.line_cons[stage][pos_start[0] - 1] -= 1
+
+        if tk in self.logic2:
+            # print tk
+            if line[pos_start[1] - 1] != tk and line[pos_start[1] + 1] != tk:
+                self.tokens += [tk]
+            if line[pos_start[1] - 1] != tk and line[pos_start[1] + 1] == tk:
+                if tk == '&':
+                    self.tokens += ['and']
+                else:
+                    self.tokens += ['or']
+            return
+
+        if self.brackets != 0:
+            def search_through_known_dicts(a):
+                for i, (n, r) in self.known_dicts_in_mkv:
+                    if i == tk and r > pos_start[1] and n == pos_start[0] and hdata == 'data':
+                        return True
+                return False
+
+            if t == token.NAME and len(self.funcs_stack) > 0 and self.funcs_stack[-1][0] == 'md5' and search_through_known_dicts(tk):
+                raise IndexCreatorValueException("Second value returned by make_key_value for sure isn't a dictionary ", self.cnt_line_nr(line, 1))
+
+        if tk == ')':
+            self.cur_brackets -= 1
+            if len(self.funcs_stack) > 0 and self.cur_brackets == self.funcs_stack[-1][1]:
+                self.tokens += [tk]
+                self.tokens += self.funcs[self.funcs_stack[-1][0]][1]
+                del self.funcs_stack[-1]
+                return
+        if tk == '(':
+            self.cur_brackets += 1
+
+        if tk in self.none:
+            self.tokens += ['None']
+            return
+
+        if t == token.NAME and tk not in self.logic and tk != hdata:
+            if tk not in self.funcs:
+                self.tokens += [hdata + '["' + tk + '"]']
+            else:
+                self.tokens += self.funcs[tk][0]
+                if tk in self.funcs_with_body:
+                    self.funcs_with_body[tk] = (
+                        self.funcs_with_body[tk][0], True)
+                self.custom_header.add(self.handle_int_imports.get(tk))
+                self.funcs_stack += [(tk, self.cur_brackets)]
+        else:
+            self.tokens += [tk]
+
+    def handle_make_value(self, t, tk, pos_start, pos_end, line):
+        self.generate_func(t, tk, pos_start, pos_end, line, 'data', 1)
+
+    def handle_make_key(self, t, tk, pos_start, pos_end, line):
+        self.generate_func(t, tk, pos_start, pos_end, line, 'key', 2)

libs/CodernityDB/lfu_cache.py

@@ -0,0 +1,150 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright 2011-2013 Codernity (http://codernity.com)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import functools
+from heapq import nsmallest
+from operator import itemgetter
+from collections import defaultdict
+
+try:
+    from collections import Counter
+except ImportError:
+    class Counter(dict):
+        'Mapping where default values are zero'
+        def __missing__(self, key):
+            return 0
+
+
+def cache1lvl(maxsize=100):
+    """
+    modified version of http://code.activestate.com/recipes/498245/
+    """
+    def decorating_function(user_function):
+        cache = {}
+        use_count = Counter()
+
+        @functools.wraps(user_function)
+        def wrapper(key, *args, **kwargs):
+            try:
+                result = cache[key]
+            except KeyError:
+                if len(cache) == maxsize:
+                    for k, _ in nsmallest(maxsize // 10 or 1,
+                                          use_count.iteritems(),
+                                          key=itemgetter(1)):
+                        del cache[k], use_count[k]
+                cache[key] = user_function(key, *args, **kwargs)
+                result = cache[key]
+                # result = user_function(obj, key, *args, **kwargs)
+            finally:
+                use_count[key] += 1
+            return result
+
+        def clear():
+            cache.clear()
+            use_count.clear()
+
+        def delete(key):
+            try:
+                del cache[key]
+                del use_count[key]
+            except KeyError:
+                return False
+            else:
+                return True
+
+        wrapper.clear = clear
+        wrapper.cache = cache
+        wrapper.delete = delete
+        return wrapper
+    return decorating_function
+
+
+def twolvl_iterator(dict):
+    for k, v in dict.iteritems():
+        for kk, vv in v.iteritems():
+            yield k, kk, vv
+
+
+def cache2lvl(maxsize=100):
+    """
+    modified version of http://code.activestate.com/recipes/498245/
+    """
+    def decorating_function(user_function):
+        cache = {}
+        use_count = defaultdict(Counter)
+
+        @functools.wraps(user_function)
+        def wrapper(*args, **kwargs):
+#            return user_function(*args, **kwargs)
+            try:
+                result = cache[args[0]][args[1]]
+            except KeyError:
+                if wrapper.cache_size == maxsize:
+                    to_delete = maxsize // 10 or 1
+                    for k1, k2, v in nsmallest(to_delete,
+                                               twolvl_iterator(use_count),
+                                               key=itemgetter(2)):
+                        del cache[k1][k2], use_count[k1][k2]
+                        if not cache[k1]:
+                            del cache[k1]
+                            del use_count[k1]
+                    wrapper.cache_size -= to_delete
+                result = user_function(*args, **kwargs)
+                try:
+                    cache[args[0]][args[1]] = result
+                except KeyError:
+                    cache[args[0]] = {args[1]: result}
+                wrapper.cache_size += 1
+            finally:
+                use_count[args[0]][args[1]] += 1
+            return result
+
+        def clear():
+            cache.clear()
+            use_count.clear()
+
+        def delete(key, inner_key=None):
+            if inner_key is not None:
+                try:
+                    del cache[key][inner_key]
+                    del use_count[key][inner_key]
+                    if not cache[key]:
+                        del cache[key]
+                        del use_count[key]
+                    wrapper.cache_size -= 1
+                except KeyError:
+                    return False
+                else:
+                    return True
+            else:
+                try:
+                    wrapper.cache_size -= len(cache[key])
+                    del cache[key]
+                    del use_count[key]
+                except KeyError:
+                    return False
+                else:
+                    return True
+
+        wrapper.clear = clear
+        wrapper.cache = cache
+        wrapper.delete = delete
+        wrapper.cache_size = 0
+        return wrapper
+    return decorating_function

libs/CodernityDB/lfu_cache_with_lock.py

@@ -0,0 +1,158 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright 2011-2013 Codernity (http://codernity.com)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import functools
+from heapq import nsmallest
+from operator import itemgetter
+from collections import defaultdict
+
+
+try:
+    from collections import Counter
+except ImportError:
+    class Counter(dict):
+        'Mapping where default values are zero'
+        def __missing__(self, key):
+            return 0
+
+
+def twolvl_iterator(dict):
+    for k, v in dict.iteritems():
+        for kk, vv in v.iteritems():
+            yield k, kk, vv
+
+
+def create_cache1lvl(lock_obj):
+    def cache1lvl(maxsize=100):
+        """
+        modified version of http://code.activestate.com/recipes/498245/
+        """
+        def decorating_function(user_function):
+            cache = {}
+            use_count = Counter()
+            lock = lock_obj()
+
+            @functools.wraps(user_function)
+            def wrapper(key, *args, **kwargs):
+                try:
+                    result = cache[key]
+                except KeyError:
+                    with lock:
+                        if len(cache) == maxsize:
+                            for k, _ in nsmallest(maxsize // 10 or 1,
+                                                  use_count.iteritems(),
+                                                  key=itemgetter(1)):
+                                del cache[k], use_count[k]
+                        cache[key] = user_function(key, *args, **kwargs)
+                        result = cache[key]
+                        use_count[key] += 1
+                else:
+                    with lock:
+                        use_count[key] += 1
+                return result
+
+            def clear():
+                cache.clear()
+                use_count.clear()
+
+            def delete(key):
+                try:
+                    del cache[key]
+                    del use_count[key]
+                    return True
+                except KeyError:
+                    return False
+
+            wrapper.clear = clear
+            wrapper.cache = cache
+            wrapper.delete = delete
+            return wrapper
+        return decorating_function
+    return cache1lvl
+
+
+def create_cache2lvl(lock_obj):
+    def cache2lvl(maxsize=100):
+        """
+        modified version of http://code.activestate.com/recipes/498245/
+        """
+        def decorating_function(user_function):
+            cache = {}
+            use_count = defaultdict(Counter)
+            lock = lock_obj()
+
+            @functools.wraps(user_function)
+            def wrapper(*args, **kwargs):
+                try:
+                    result = cache[args[0]][args[1]]
+                except KeyError:
+                    with lock:
+                        if wrapper.cache_size == maxsize:
+                            to_delete = maxsize / 10 or 1
+                            for k1, k2, v in nsmallest(to_delete,
+                                                       twolvl_iterator(
+                                                           use_count),
+                                                       key=itemgetter(2)):
+                                del cache[k1][k2], use_count[k1][k2]
+                                if not cache[k1]:
+                                    del cache[k1]
+                                    del use_count[k1]
+                            wrapper.cache_size -= to_delete
+                        result = user_function(*args, **kwargs)
+                        try:
+                            cache[args[0]][args[1]] = result
+                        except KeyError:
+                            cache[args[0]] = {args[1]: result}
+                        use_count[args[0]][args[1]] += 1
+                        wrapper.cache_size += 1
+                else:
+                    use_count[args[0]][args[1]] += 1
+                return result
+
+            def clear():
+                cache.clear()
+                use_count.clear()
+
+            def delete(key, *args):
+                if args:
+                    try:
+                        del cache[key][args[0]]
+                        del use_count[key][args[0]]
+                        if not cache[key]:
+                            del cache[key]
+                            del use_count[key]
+                        wrapper.cache_size -= 1
+                        return True
+                    except KeyError:
+                        return False
+                else:
+                    try:
+                        wrapper.cache_size -= len(cache[key])
+                        del cache[key]
+                        del use_count[key]
+                        return True
+                    except KeyError:
+                        return False
+
+            wrapper.clear = clear
+            wrapper.cache = cache
+            wrapper.delete = delete
+            wrapper.cache_size = 0
+            return wrapper
+        return decorating_function
+    return cache2lvl

libs/CodernityDB/migrate.py

@@ -0,0 +1,45 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright 2011-2013 Codernity (http://codernity.com)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from CodernityDB.database import Database
+import shutil
+import os
+
+
+def migrate(source, destination):
+    """
+    Very basic for now
+    """
+    dbs = Database(source)
+    dbt = Database(destination)
+    dbs.open()
+    dbt.create()
+    dbt.close()
+    for curr in os.listdir(os.path.join(dbs.path, '_indexes')):
+        if curr != '00id.py':
+            shutil.copyfile(os.path.join(dbs.path, '_indexes', curr),
+                            os.path.join(dbt.path, '_indexes', curr))
+    dbt.open()
+    for c in dbs.all('id'):
+        del c['_rev']
+        dbt.insert(c)
+    return True
+
+
+if __name__ == '__main__':
+    import sys
+    migrate(sys.argv[1], sys.argv[2])

libs/CodernityDB/misc.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright 2011-2013 Codernity (http://codernity.com)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from random import getrandbits, randrange
+import uuid
+
+
+class NONE:
+    """
+    It's inteded to be None but different,
+    for internal use only!
+    """
+    pass
+
+
+def random_hex_32():
+    return uuid.UUID(int=getrandbits(128), version=4).hex
+
+
+def random_hex_4(*args, **kwargs):
+    return '%04x' % randrange(256 ** 2)

libs/CodernityDB/patch.py

@@ -0,0 +1,99 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright 2011-2013 Codernity (http://codernity.com)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from CodernityDB.misc import NONE
+
+
+def __patch(obj, name, new):
+    n = NONE()
+    orig = getattr(obj, name, n)
+    if orig is not n:
+        if orig == new:
+            raise Exception("Shouldn't happen, new and orig are the same")
+        setattr(obj, name, new)
+    return
+
+
+def patch_cache_lfu(lock_obj):
+    """
+    Patnches cache mechanizm to be thread safe (gevent ones also)
+
+    .. note::
+
+       It's internal CodernityDB mechanizm, it will be called when needed
+
+    """
+    import lfu_cache
+    import lfu_cache_with_lock
+    lfu_lock1lvl = lfu_cache_with_lock.create_cache1lvl(lock_obj)
+    lfu_lock2lvl = lfu_cache_with_lock.create_cache2lvl(lock_obj)
+    __patch(lfu_cache, 'cache1lvl', lfu_lock1lvl)
+    __patch(lfu_cache, 'cache2lvl', lfu_lock2lvl)
+
+
+def patch_cache_rr(lock_obj):
+    """
+    Patches cache mechanizm to be thread safe (gevent ones also)
+
+    .. note::
+
+       It's internal CodernityDB mechanizm, it will be called when needed
+
+    """
+    import rr_cache
+    import rr_cache_with_lock
+    rr_lock1lvl = rr_cache_with_lock.create_cache1lvl(lock_obj)
+    rr_lock2lvl = rr_cache_with_lock.create_cache2lvl(lock_obj)
+    __patch(rr_cache, 'cache1lvl', rr_lock1lvl)
+    __patch(rr_cache, 'cache2lvl', rr_lock2lvl)
+
+
+def patch_flush_fsync(db_obj):
+    """
+    Will always execute index.fsync after index.flush.
+
+    .. note::
+
+       It's for advanced users, use when you understand difference between `flush` and `fsync`, and when you definitely need that.
+
+    It's important to call it **AFTER** database has all indexes etc (after db.create or db.open)
+
+    Example usage::
+
+        ...
+        db = Database('/tmp/patch_demo')
+        db.create()
+        patch_flush_fsync(db)
+        ...
+
+    """
+
+    def always_fsync(ind_obj):
+        def _inner():
+            ind_obj.orig_flush()
+            ind_obj.fsync()
+        return _inner
+
+    for index in db_obj.indexes:
+        setattr(index, 'orig_flush', index.flush)
+        setattr(index, 'flush', always_fsync(index))
+
+    setattr(db_obj, 'orig_flush', db_obj.flush)
+    setattr(db_obj, 'flush', always_fsync(db_obj))
+
+    return

libs/CodernityDB/rr_cache.py

@@ -0,0 +1,112 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright 2011-2013 Codernity (http://codernity.com)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import functools
+from random import choice
+
+
+def cache1lvl(maxsize=100):
+    def decorating_function(user_function):
+        cache1lvl = {}
+
+        @functools.wraps(user_function)
+        def wrapper(key, *args, **kwargs):
+            try:
+                result = cache1lvl[key]
+            except KeyError:
+                if len(cache1lvl) == maxsize:
+                    for i in xrange(maxsize // 10 or 1):
+                        del cache1lvl[choice(cache1lvl.keys())]
+                cache1lvl[key] = user_function(key, *args, **kwargs)
+                result = cache1lvl[key]
+#                result = user_function(obj, key, *args, **kwargs)
+            return result
+
+        def clear():
+            cache1lvl.clear()
+
+        def delete(key):
+            try:
+                del cache1lvl[key]
+                return True
+            except KeyError:
+                return False
+
+        wrapper.clear = clear
+        wrapper.cache = cache1lvl
+        wrapper.delete = delete
+        return wrapper
+    return decorating_function
+
+
+def cache2lvl(maxsize=100):
+    def decorating_function(user_function):
+        cache = {}
+
+        @functools.wraps(user_function)
+        def wrapper(*args, **kwargs):
+#            return user_function(*args, **kwargs)
+            try:
+                result = cache[args[0]][args[1]]
+            except KeyError:
+#                print wrapper.cache_size
+                if wrapper.cache_size == maxsize:
+                    to_delete = maxsize // 10 or 1
+                    for i in xrange(to_delete):
+                        key1 = choice(cache.keys())
+                        key2 = choice(cache[key1].keys())
+                        del cache[key1][key2]
+                        if not cache[key1]:
+                            del cache[key1]
+                    wrapper.cache_size -= to_delete
+#                print wrapper.cache_size
+                result = user_function(*args, **kwargs)
+                try:
+                    cache[args[0]][args[1]] = result
+                except KeyError:
+                    cache[args[0]] = {args[1]: result}
+                wrapper.cache_size += 1
+            return result
+
+        def clear():
+            cache.clear()
+            wrapper.cache_size = 0
+
+        def delete(key, inner_key=None):
+            if inner_key:
+                try:
+                    del cache[key][inner_key]
+                    if not cache[key]:
+                        del cache[key]
+                    wrapper.cache_size -= 1
+                    return True
+                except KeyError:
+                    return False
+            else:
+                try:
+                    wrapper.cache_size -= len(cache[key])
+                    del cache[key]
+                    return True
+                except KeyError:
+                    return False
+
+        wrapper.clear = clear
+        wrapper.cache = cache
+        wrapper.delete = delete
+        wrapper.cache_size = 0
+        return wrapper
+    return decorating_function

libs/CodernityDB/rr_cache_with_lock.py

@@ -0,0 +1,117 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright 2011-2013 Codernity (http://codernity.com)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import functools
+from random import choice
+
+
+def create_cache1lvl(lock_obj):
+    def cache1lvl(maxsize=100):
+        def decorating_function(user_function):
+            cache = {}
+            lock = lock_obj()
+
+            @functools.wraps(user_function)
+            def wrapper(key, *args, **kwargs):
+                try:
+                    result = cache[key]
+                except KeyError:
+                    with lock:
+                        if len(cache) == maxsize:
+                            for i in xrange(maxsize // 10 or 1):
+                                del cache[choice(cache.keys())]
+                        cache[key] = user_function(key, *args, **kwargs)
+                        result = cache[key]
+                return result
+
+            def clear():
+                cache.clear()
+
+            def delete(key):
+                try:
+                    del cache[key]
+                    return True
+                except KeyError:
+                    return False
+
+            wrapper.clear = clear
+            wrapper.cache = cache
+            wrapper.delete = delete
+            return wrapper
+        return decorating_function
+    return cache1lvl
+
+
+def create_cache2lvl(lock_obj):
+    def cache2lvl(maxsize=100):
+        def decorating_function(user_function):
+            cache = {}
+            lock = lock_obj()
+
+            @functools.wraps(user_function)
+            def wrapper(*args, **kwargs):
+                try:
+                    result = cache[args[0]][args[1]]
+                except KeyError:
+                    with lock:
+                        if wrapper.cache_size == maxsize:
+                            to_delete = maxsize // 10 or 1
+                            for i in xrange(to_delete):
+                                key1 = choice(cache.keys())
+                                key2 = choice(cache[key1].keys())
+                                del cache[key1][key2]
+                                if not cache[key1]:
+                                    del cache[key1]
+                            wrapper.cache_size -= to_delete
+                        result = user_function(*args, **kwargs)
+                        try:
+                            cache[args[0]][args[1]] = result
+                        except KeyError:
+                            cache[args[0]] = {args[1]: result}
+                        wrapper.cache_size += 1
+                return result
+
+            def clear():
+                cache.clear()
+                wrapper.cache_size = 0
+
+            def delete(key, *args):
+                if args:
+                    try:
+                        del cache[key][args[0]]
+                        if not cache[key]:
+                            del cache[key]
+                        wrapper.cache_size -= 1
+                        return True
+                    except KeyError:
+                        return False
+                else:
+                    try:
+                        wrapper.cache_size -= len(cache[key])
+                        del cache[key]
+                        return True
+                    except KeyError:
+                        return False
+
+            wrapper.clear = clear
+            wrapper.cache = cache
+            wrapper.delete = delete
+            wrapper.cache_size = 0
+            return wrapper
+        return decorating_function
+    return cache2lvl

libs/CodernityDB/sharded_hash.py

@@ -0,0 +1,146 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright 2011-2013 Codernity (http://codernity.com)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from CodernityDB.hash_index import UniqueHashIndex, HashIndex
+from CodernityDB.sharded_index import ShardedIndex
+from CodernityDB.index import IndexPreconditionsException
+
+from random import getrandbits
+import uuid
+
+
+class IU_ShardedUniqueHashIndex(ShardedIndex):
+
+    custom_header = """import uuid
+from random import getrandbits
+from CodernityDB.sharded_index import ShardedIndex
+"""
+
+    def __init__(self, db_path, name, *args, **kwargs):
+        if kwargs.get('sh_nums', 0) > 255:
+            raise IndexPreconditionsException("Too many shards")
+        kwargs['ind_class'] = UniqueHashIndex
+        super(IU_ShardedUniqueHashIndex, self).__init__(db_path,
+                                                        name, *args, **kwargs)
+        self.patchers.append(self.wrap_insert_id_index)
+
+    @staticmethod
+    def wrap_insert_id_index(db_obj, clean=False):
+        def _insert_id_index(_rev, data):
+            """
+            Performs insert on **id** index.
+            """
+            _id, value = db_obj.id_ind.make_key_value(data)  # may be improved
+            trg_shard = _id[:2]
+            storage = db_obj.id_ind.shards_r[trg_shard].storage
+            start, size = storage.insert(value)
+            db_obj.id_ind.insert(_id, _rev, start, size)
+            return _id
+        if not clean:
+            if hasattr(db_obj, '_insert_id_index_orig'):
+                raise IndexPreconditionsException(
+                    "Already patched, something went wrong")
+            setattr(db_obj, "_insert_id_index_orig", db_obj._insert_id_index)
+            setattr(db_obj, "_insert_id_index", _insert_id_index)
+        else:
+            setattr(db_obj, "_insert_id_index", db_obj._insert_id_index_orig)
+            delattr(db_obj, "_insert_id_index_orig")
+
+    def create_key(self):
+        h = uuid.UUID(int=getrandbits(128), version=4).hex
+        trg = self.last_used + 1
+        if trg >= self.sh_nums:
+            trg = 0
+        self.last_used = trg
+        h = '%02x%30s' % (trg, h[2:])
+        return h
+
+    def delete(self, key, *args, **kwargs):
+        trg_shard = key[:2]
+        op = self.shards_r[trg_shard]
+        return op.delete(key, *args, **kwargs)
+
+    def update(self, key, *args, **kwargs):
+        trg_shard = key[:2]
+        self.last_used = int(trg_shard, 16)
+        op = self.shards_r[trg_shard]
+        return op.update(key, *args, **kwargs)
+
+    def insert(self, key, *args, **kwargs):
+        trg_shard = key[:2]  # in most cases it's in create_key BUT not always
+        self.last_used = int(key[:2], 16)
+        op = self.shards_r[trg_shard]
+        return op.insert(key, *args, **kwargs)
+
+    def get(self, key, *args, **kwargs):
+        trg_shard = key[:2]
+        self.last_used = int(trg_shard, 16)
+        op = self.shards_r[trg_shard]
+        return op.get(key, *args, **kwargs)
+
+
+class ShardedUniqueHashIndex(IU_ShardedUniqueHashIndex):
+
+    # allow unique hash to be used directly
+    custom_header = 'from CodernityDB.sharded_hash import IU_ShardedUniqueHashIndex'
+
+    pass
+
+
+class IU_ShardedHashIndex(ShardedIndex):
+
+    custom_header = """from CodernityDB.sharded_index import ShardedIndex"""
+
+    def __init__(self, db_path, name, *args, **kwargs):
+        kwargs['ind_class'] = HashIndex
+        super(IU_ShardedHashIndex, self).__init__(db_path, name, *
+                                                  args, **kwargs)
+
+    def calculate_shard(self, key):
+        """
+        Must be implemented. It has to return shard to be used by key
+
+        :param key: key
+        :returns: target shard
+        :rtype: int
+        """
+        raise NotImplementedError()
+
+    def delete(self, doc_id, key, *args, **kwargs):
+        trg_shard = self.calculate_shard(key)
+        op = self.shards_r[trg_shard]
+        return op.delete(doc_id, key, *args, **kwargs)
+
+    def insert(self, doc_id, key, *args, **kwargs):
+        trg_shard = self.calculate_shard(key)
+        op = self.shards_r[trg_shard]
+        return op.insert(doc_id, key, *args, **kwargs)
+
+    def update(self, doc_id, key, *args, **kwargs):
+        trg_shard = self.calculate_shard(key)
+        op = self.shards_r[trg_shard]
+        return op.insert(doc_id, key, *args, **kwargs)
+
+    def get(self, key, *args, **kwargs):
+        trg_shard = self.calculate_shard(key)
+        op = self.shards_r[trg_shard]
+        return op.get(key, *args, **kwargs)
+
+
+class ShardedHashIndex(IU_ShardedHashIndex):
+    pass

libs/CodernityDB/sharded_index.py

@@ -0,0 +1,112 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright 2011-2013 Codernity (http://codernity.com)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from CodernityDB.index import Index
+# from CodernityDB.env import cdb_environment
+# import warnings
+
+
+class ShardedIndex(Index):
+
+    def __init__(self, db_path, name, *args, **kwargs):
+        """
+        There are 3 additional parameters. You have to hardcode them in your custom class. **NEVER** use directly
+
+        :param int sh_nums: how many shards should be
+        :param class ind_class: Index class to use (HashIndex or your custom one)
+        :param bool use_make_keys: if True, `make_key`, and `make_key_value` will be overriden with those from first shard
+
+        The rest parameters are passed straight to `ind_class` shards.
+
+        """
+        super(ShardedIndex, self).__init__(db_path, name)
+        try:
+            self.sh_nums = kwargs.pop('sh_nums')
+        except KeyError:
+            self.sh_nums = 5
+        try:
+            ind_class = kwargs.pop('ind_class')
+        except KeyError:
+            raise Exception("ind_class must be given")
+        else:
+            # if not isinstance(ind_class, basestring):
+            #     ind_class = ind_class.__name__
+            self.ind_class = ind_class
+        if 'use_make_keys' in kwargs:
+            self.use_make_keys = kwargs.pop('use_make_keys')
+        else:
+            self.use_make_keys = False
+        self._set_shard_datas(*args, **kwargs)
+        self.patchers = []  # database object patchers
+
+    def _set_shard_datas(self, *args, **kwargs):
+        self.shards = {}
+        self.shards_r = {}
+#        ind_class = globals()[self.ind_class]
+        ind_class = self.ind_class
+        i = 0
+        for sh_name in [self.name + str(x) for x in xrange(self.sh_nums)]:
+            # dict is better than list in that case
+            self.shards[i] = ind_class(self.db_path, sh_name, *args, **kwargs)
+            self.shards_r['%02x' % i] = self.shards[i]
+            self.shards_r[i] = self.shards[i]
+            i += 1
+
+        if not self.use_make_keys:
+            self.make_key = self.shards[0].make_key
+            self.make_key_value = self.shards[0].make_key_value
+
+        self.last_used = 0
+
+    @property
+    def storage(self):
+        st = self.shards[self.last_used].storage
+        return st
+
+    def __getattr__(self, name):
+        return getattr(self.shards[self.last_used], name)
+
+    def open_index(self):
+        for curr in self.shards.itervalues():
+            curr.open_index()
+
+    def create_index(self):
+        for curr in self.shards.itervalues():
+            curr.create_index()
+
+    def destroy(self):
+        for curr in self.shards.itervalues():
+            curr.destroy()
+
+    def compact(self):
+        for curr in self.shards.itervalues():
+            curr.compact()
+
+    def reindex(self):
+        for curr in self.shards.itervalues():
+            curr.reindex()
+
+    def all(self, *args, **kwargs):
+        for curr in self.shards.itervalues():
+            for now in curr.all(*args, **kwargs):
+                yield now
+
+    def get_many(self, *args, **kwargs):
+        for curr in self.shards.itervalues():
+            for now in curr.get_many(*args, **kwargs):
+                yield now

libs/CodernityDB/storage.py

@@ -0,0 +1,153 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright 2011-2013 Codernity (http://codernity.com)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import struct
+import shutil
+import marshal
+import io
+
+
+try:
+    from CodernityDB import __version__
+except ImportError:
+    from __init__ import __version__
+
+
+class StorageException(Exception):
+    pass
+
+
+class DummyStorage(object):
+    """
+    Storage mostly used to fake real storage
+    """
+
+    def create(self, *args, **kwargs):
+        pass
+
+    def open(self, *args, **kwargs):
+        pass
+
+    def close(self, *args, **kwargs):
+        pass
+
+    def data_from(self, *args, **kwargs):
+        pass
+
+    def data_to(self, *args, **kwargs):
+        pass
+
+    def save(self, *args, **kwargs):
+        return 0, 0
+
+    def insert(self, *args, **kwargs):
+        return self.save(*args, **kwargs)
+
+    def update(self, *args, **kwargs):
+        return 0, 0
+
+    def get(self, *args, **kwargs):
+        return None
+
+    # def compact(self, *args, **kwargs):
+    #     pass
+
+    def fsync(self, *args, **kwargs):
+        pass
+
+    def flush(self, *args, **kwargs):
+        pass
+
+
+class IU_Storage(object):
+
+    __version__ = __version__
+
+    def __init__(self, db_path, name='main'):
+        self.db_path = db_path
+        self.name = name
+        self._header_size = 100
+
+    def create(self):
+        if os.path.exists(os.path.join(self.db_path, self.name + "_stor")):
+            raise IOError("Storage already exists!")
+        with io.open(os.path.join(self.db_path, self.name + "_stor"), 'wb') as f:
+            f.write(struct.pack("10s90s", self.__version__, '|||||'))
+            f.close()
+        self._f = io.open(os.path.join(
+            self.db_path, self.name + "_stor"), 'r+b', buffering=0)
+        self.flush()
+        self._f.seek(0, 2)
+
+    def open(self):
+        if not os.path.exists(os.path.join(self.db_path, self.name + "_stor")):
+            raise IOError("Storage doesn't exists!")
+        self._f = io.open(os.path.join(
+            self.db_path, self.name + "_stor"), 'r+b', buffering=0)
+        self.flush()
+        self._f.seek(0, 2)
+
+    def destroy(self):
+        os.unlink(os.path.join(self.db_path, self.name + '_stor'))
+
+    def close(self):
+        self._f.close()
+        # self.flush()
+        # self.fsync()
+
+    def data_from(self, data):
+        return marshal.loads(data)
+
+    def data_to(self, data):
+        return marshal.dumps(data)
+
+    def save(self, data):
+        s_data = self.data_to(data)
+        self._f.seek(0, 2)
+        start = self._f.tell()
+        size = len(s_data)
+        self._f.write(s_data)
+        self.flush()
+        return start, size
+
+    def insert(self, data):
+        return self.save(data)
+
+    def update(self, data):
+        return self.save(data)
+
+    def get(self, start, size, status='c'):
+        if status == 'd':
+            return None
+        else:
+            self._f.seek(start)
+            return self.data_from(self._f.read(size))
+
+    def flush(self):
+        self._f.flush()
+
+    def fsync(self):
+        os.fsync(self._f.fileno())
+
+
+# classes for public use, done in this way because of
+# generation static files with indexes (_index directory)
+
+
+class Storage(IU_Storage):
+    pass