diff --git a/couchpotato/__init__.py b/couchpotato/__init__.py index 6b8cfd36..dba45861 100644 --- a/couchpotato/__init__.py +++ b/couchpotato/__init__.py @@ -49,7 +49,11 @@ def addView(route, func, static = False): def get_session(): - return Env.getSession() + return None + + +def get_db(): + return Env.get('db') # Web view diff --git a/couchpotato/core/_base/_core/main.py b/couchpotato/core/_base/_core/main.py index 02e21f2d..22b5a91b 100644 --- a/couchpotato/core/_base/_core/main.py +++ b/couchpotato/core/_base/_core/main.py @@ -42,11 +42,11 @@ class Core(Plugin): addEvent('app.shutdown', self.shutdown) addEvent('app.restart', self.restart) - addEvent('app.load', self.launchBrowser, priority = 1) + addEvent('app.load2', self.launchBrowser, priority = 1) addEvent('app.base_url', self.createBaseUrl) addEvent('app.api_url', self.createApiUrl) addEvent('app.version', self.version) - addEvent('app.load', self.checkDataDir) + addEvent('app.load2', self.checkDataDir) addEvent('setting.save.core.password', self.md5Password) addEvent('setting.save.core.api_key', self.checkApikey) diff --git a/couchpotato/core/_base/clientscript/main.py b/couchpotato/core/_base/clientscript/main.py index 248d2bc5..1c9d82a6 100644 --- a/couchpotato/core/_base/clientscript/main.py +++ b/couchpotato/core/_base/clientscript/main.py @@ -74,7 +74,7 @@ class ClientScript(Plugin): addEvent('clientscript.get_scripts', self.getScripts) if not Env.get('dev'): - addEvent('app.load', self.minify) + addEvent('app.load2', self.minify) self.addCore() diff --git a/couchpotato/core/_base/desktop/main.py b/couchpotato/core/_base/desktop/main.py index c3beff17..21499159 100644 --- a/couchpotato/core/_base/desktop/main.py +++ b/couchpotato/core/_base/desktop/main.py @@ -25,7 +25,7 @@ if Env.get('desktop'): # Events to desktop addEvent('app.after_shutdown', desktop.afterShutdown) - addEvent('app.load', desktop.onAppLoad, priority = 110) + addEvent('app.load2', desktop.onAppLoad, priority = 110) def onClose(self, event): return fireEvent('app.shutdown', single = True) diff --git a/couchpotato/core/_base/updater/main.py b/couchpotato/core/_base/updater/main.py index ef595ad7..aad4eee2 100644 --- a/couchpotato/core/_base/updater/main.py +++ b/couchpotato/core/_base/updater/main.py @@ -33,8 +33,8 @@ class Updater(Plugin): else: self.updater = SourceUpdater() - addEvent('app.load', self.logVersion, priority = 10000) - addEvent('app.load', self.setCrons) + addEvent('app.load2', self.logVersion, priority = 10000) + addEvent('app.load2', self.setCrons) addEvent('updater.info', self.info) addApiView('updater.info', self.info, docs = { diff --git a/couchpotato/core/downloaders/rtorrent/main.py b/couchpotato/core/downloaders/rtorrent/main.py index 27177756..23b7a25c 100755 --- a/couchpotato/core/downloaders/rtorrent/main.py +++ b/couchpotato/core/downloaders/rtorrent/main.py @@ -23,7 +23,7 @@ class rTorrent(Downloader): def __init__(self): super(rTorrent, self).__init__() - addEvent('app.load', self.migrate) + addEvent('app.load2', self.migrate) def migrate(self): diff --git a/couchpotato/core/media/__init__.py b/couchpotato/core/media/__init__.py index 512c52c2..db87aa75 100644 --- a/couchpotato/core/media/__init__.py +++ b/couchpotato/core/media/__init__.py @@ -1,9 +1,11 @@ import traceback -from couchpotato import get_session +from couchpotato import get_session, get_db, CPLog from couchpotato.core.event import addEvent, fireEventAsync, fireEvent from couchpotato.core.plugins.base import Plugin from couchpotato.core.settings.model import Media +log = CPLog(__name__) + class MediaBase(Plugin): @@ -24,20 +26,19 @@ class MediaBase(Plugin): def getType(self): return self._type - def createOnComplete(self, id): + def createOnComplete(self, media_id): def onComplete(): try: - db = get_session() - media = db.query(Media).filter_by(id = id).first() - media_dict = media.to_dict(self.default_dict) - event_name = '%s.searcher.single' % media.type + db = get_db() + media = db.get('id', media_id) + event_name = '%s.searcher.single' % media.get('type') - fireEvent(event_name, media_dict, on_complete = self.createNotifyFront(id)) + fireEvent(event_name, media, on_complete = self.createNotifyFront(media_id)) except: log.error('Failed creating onComplete: %s', traceback.format_exc()) finally: - db.close() + pass #db.close() return onComplete @@ -45,15 +46,14 @@ class MediaBase(Plugin): def notifyFront(): try: - db = get_session() - media = db.query(Media).filter_by(id = media_id).first() - media_dict = media.to_dict(self.default_dict) - event_name = '%s.update' % media.type + db = get_db() + media = db.get('id', media_id) + event_name = '%s.update' % media.get('type') - fireEvent('notify.frontend', type = event_name, data = media_dict) + fireEvent('notify.frontend', type = event_name, data = media) except: log.error('Failed creating onComplete: %s', traceback.format_exc()) finally: - db.close() + pass #db.close() return notifyFront diff --git a/couchpotato/core/media/_base/media/index.py b/couchpotato/core/media/_base/media/index.py new file mode 100644 index 00000000..bd90fcf4 --- /dev/null +++ b/couchpotato/core/media/_base/media/index.py @@ -0,0 +1,60 @@ +from CodernityDB.hash_index import HashIndex +from CodernityDB.tree_index import MultiTreeBasedIndex + + +class MediaIMDBIndex(HashIndex): + + def __init__(self, *args, **kwargs): + kwargs['key_format'] = 'I' + super(MediaIMDBIndex, self).__init__(*args, **kwargs) + + def make_key(self, key): + return int(key.strip('t')) + + def make_key_value(self, data): + if data.get('type') == 'media' and data.get('identifier'): + return int(data['identifier'].strip('t')), None + + +class MediaStatusIndex(HashIndex): + + def __init__(self, *args, **kwargs): + kwargs['key_format'] = 's' + super(MediaStatusIndex, self).__init__(*args, **kwargs) + + def make_key(self, key): + return int(key.strip('t')) + + def make_key_value(self, data): + if data.get('type') == 'media' and data.get('identifier'): + return int(data['identifier'].strip('t')), None + + +class TitleIndex(MultiTreeBasedIndex): + + custom_header = """from CodernityDB.tree_index import MultiTreeBasedIndex +from itertools import izip""" + + def __init__(self, *args, **kwargs): + kwargs['key_format'] = '32s' + super(TitleIndex, self).__init__(*args, **kwargs) + self.__l = kwargs.get('w_len', 2) + + def make_key_value(self, data): + + if data.get('type') == 'title' and len(data.get('title', '')) > 0: + + out = set() + title = data.get('title').lower() + l = self.__l + max_l = len(title) + for x in xrange(l - 1, max_l): + m = (title, ) + for y in xrange(0, x): + m += (title[y + 1:],) + out.update(set(''.join(x).rjust(32, '_').lower() for x in izip(*m))) #ignore import error + + return out, {'media_id': data.get('media_id')} + + def make_key(self, key): + return key.rjust(32, '_').lower() diff --git a/couchpotato/core/media/_base/media/main.py b/couchpotato/core/media/_base/media/main.py index 5345b07c..9a50ffe8 100644 --- a/couchpotato/core/media/_base/media/main.py +++ b/couchpotato/core/media/_base/media/main.py @@ -1,11 +1,12 @@ import traceback -from couchpotato import get_session, tryInt +from couchpotato import get_session, tryInt, get_db from couchpotato.api import addApiView from couchpotato.core.event import fireEvent, fireEventAsync, addEvent from couchpotato.core.helpers.encoding import toUnicode from couchpotato.core.helpers.variable import mergeDicts, splitString, getImdb, getTitle from couchpotato.core.logger import CPLog from couchpotato.core.media import MediaBase +from .index import MediaIMDBIndex, TitleIndex, MediaStatusIndex from couchpotato.core.settings.model import Library, LibraryTitle, Release, \ Media from sqlalchemy.orm import joinedload_all @@ -60,16 +61,40 @@ class MediaPlugin(MediaBase): addApiView('media.available_chars', self.charView) - addEvent('app.load', self.addSingleRefreshView) - addEvent('app.load', self.addSingleListView) - addEvent('app.load', self.addSingleCharView) - addEvent('app.load', self.addSingleDeleteView) + addEvent('database.setup', self.databaseSetup) + + addEvent('app.load2', self.addSingleRefreshView) + addEvent('app.load2', self.addSingleListView) + addEvent('app.load2', self.addSingleCharView) + addEvent('app.load2', self.addSingleDeleteView) addEvent('media.get', self.get) addEvent('media.list', self.list) addEvent('media.delete', self.delete) addEvent('media.restatus', self.restatus) + def databaseSetup(self): + + db = get_db() + + try: + db.add_index(MediaIMDBIndex(db.path, 'media')) + except: + log.debug('Index already exists') + db.update_index(MediaIMDBIndex(db.path, 'media')) + + try: + db.add_index(TitleIndex(db.path, 'media_title')) + except: + log.debug('Index already exists') + db.update_index(TitleIndex(db.path, 'media_title')) + + try: + db.add_index(MediaStatusIndex(db.path, 'media_status')) + except: + log.debug('Index already exists') + db.update_index(MediaStatusIndex(db.path, 'media_status')) + def refresh(self, id = '', **kwargs): handlers = [] ids = splitString(id) @@ -87,24 +112,23 @@ class MediaPlugin(MediaBase): 'success': True, } - def createRefreshHandler(self, id): - db = get_session() + def createRefreshHandler(self, media_id): - media = db.query(Media).filter_by(id = id).first() + try: + media = get_db().get('id', media_id) - if media: - - default_title = getTitle(media.library) - identifier = media.library.identifier - event = 'library.update.%s' % media.type + default_title = getTitle(media_id) + event = 'library.update.%s' % media.get('type') def handler(): - fireEvent(event, identifier = identifier, default_title = default_title, on_complete = self.createOnComplete(id)) + fireEvent(event, identifier = media.get('identifier'), default_title = default_title, on_complete = self.createOnComplete(media_id)) - db.close() + if handler: + return handler + + except: + log.error('Refresh handler for non existing media: %s', traceback.format_exc()) - if handler: - return handler def addSingleRefreshView(self): @@ -113,20 +137,19 @@ class MediaPlugin(MediaBase): def get(self, media_id): - db = get_session() + db = get_db() imdb_id = getImdb(str(media_id)) if imdb_id: - m = db.query(Media).filter(Media.library.has(identifier = imdb_id)).first() + m = db.get('media', imdb_id, with_doc = True)['doc'] else: - m = db.query(Media).filter_by(id = media_id).first() + m = db.get('id', media_id) results = None if m: - results = m.to_dict(self.default_dict) + results = db.run('media', 'to_dict', m, self.default_dict) - db.close() return results def getView(self, id = None, **kwargs): @@ -262,7 +285,7 @@ class MediaPlugin(MediaBase): 'releases_count': releases_count.get(media_id), })) - db.close() + pass #db.close() return total_count, movies def listView(self, **kwargs): @@ -356,7 +379,7 @@ class MediaPlugin(MediaBase): if len(chars) == 25: break - db.close() + pass #db.close() return ''.join(sorted(chars)) def charView(self, **kwargs): @@ -428,7 +451,7 @@ class MediaPlugin(MediaBase): log.error('Failed deleting media: %s', traceback.format_exc()) db.rollback() finally: - db.close() + pass #db.close() return True @@ -481,5 +504,5 @@ class MediaPlugin(MediaBase): log.error('Failed restatus: %s', traceback.format_exc()) db.rollback() finally: - db.close() + pass #db.close() diff --git a/couchpotato/core/media/_base/search/main.py b/couchpotato/core/media/_base/search/main.py index 81897b5f..6f0f5172 100644 --- a/couchpotato/core/media/_base/search/main.py +++ b/couchpotato/core/media/_base/search/main.py @@ -25,7 +25,7 @@ class Search(Plugin): }"""} }) - addEvent('app.load', self.addSingleSearches) + addEvent('app.load2', self.addSingleSearches) def search(self, q = '', types = None, **kwargs): diff --git a/couchpotato/core/media/_base/searcher/base.py b/couchpotato/core/media/_base/searcher/base.py index 5322d850..60799bf0 100644 --- a/couchpotato/core/media/_base/searcher/base.py +++ b/couchpotato/core/media/_base/searcher/base.py @@ -28,7 +28,7 @@ class SearcherBase(Plugin): fireEvent('schedule.cron', '%s.searcher.all' % _type, self.searchAll, day = self.conf('cron_day'), hour = self.conf('cron_hour'), minute = self.conf('cron_minute')) - addEvent('app.load', setCrons) + addEvent('app.load2', setCrons) addEvent('setting.save.%s_searcher.cron_day.after' % _type, setCrons) addEvent('setting.save.%s_searcher.cron_hour.after' % _type, setCrons) addEvent('setting.save.%s_searcher.cron_minute.after' % _type, setCrons) diff --git a/couchpotato/core/media/movie/_base/main.py b/couchpotato/core/media/movie/_base/main.py index a7ecf2d2..306766c4 100644 --- a/couchpotato/core/media/movie/_base/main.py +++ b/couchpotato/core/media/movie/_base/main.py @@ -1,5 +1,5 @@ import traceback -from couchpotato import get_session +from couchpotato import get_session, get_db from couchpotato.api import addApiView from couchpotato.core.event import fireEvent, fireEventAsync, addEvent from couchpotato.core.helpers.encoding import toUnicode @@ -62,78 +62,78 @@ class MovieBase(MovieTypeBase): except: pass - library = fireEvent('library.add.movie', single = True, attrs = params, update_after = update_library) - - # Status - status_active, snatched_status, ignored_status, done_status, downloaded_status = \ - fireEvent('status.get', ['active', 'snatched', 'ignored', 'done', 'downloaded'], single = True) + # library = fireEvent('library.add.movie', single = True, attrs = params, update_after = update_library) + info = fireEvent('movie.info', merge = True, extended = False, identifier = params.get('identifier')) default_profile = fireEvent('profile.default', single = True) cat_id = params.get('category_id') try: - db = get_session() - m = db.query(Media).filter_by(library_id = library.get('id')).first() + db = get_db() + + new = False + try: + m = db.get('movie', params.get('identifier'), with_doc = True)['doc'] + except: + new = True + m = db.insert({ + 'type': 'movie', + 'identifier': params.get('identifier'), + 'status': status_id if status_id else 'active', + 'profile_id': params.get('profile_id', default_profile.get('id')), + 'category_id': tryInt(cat_id) if cat_id is not None and tryInt(cat_id) > 0 else None, + }) + added = True do_search = False search_after = search_after and self.conf('search_on_add', section = 'moviesearcher') - if not m: - m = Media( - library_id = library.get('id'), - profile_id = params.get('profile_id', default_profile.get('id')), - status_id = status_id if status_id else status_active.get('id'), - category_id = tryInt(cat_id) if cat_id is not None and tryInt(cat_id) > 0 else None, - ) - db.add(m) - db.commit() - + if new: onComplete = None if search_after: - onComplete = self.createOnComplete(m.id) + onComplete = self.createOnComplete(m['_id']) - fireEventAsync('library.update.movie', params.get('identifier'), default_title = params.get('title', ''), on_complete = onComplete) + # fireEventAsync('library.update.movie', params.get('identifier'), default_title = params.get('title', ''), on_complete = onComplete) search_after = False elif force_readd: # Clean snatched history - for release in m.releases: - if release.status_id in [downloaded_status.get('id'), snatched_status.get('id'), done_status.get('id')]: + for release in db.run('release', 'for_media', m['_id']): + if release.get('status') in ['downloaded', 'snatched', 'done']: if params.get('ignore_previous', False): - release.status_id = ignored_status.get('id') + release['status'] = 'ignored' + db.update(release) else: - fireEvent('release.delete', release.id, single = True) + fireEvent('release.delete', release['_id'], single = True) - m.profile_id = params.get('profile_id', default_profile.get('id')) - m.category_id = tryInt(cat_id) if cat_id is not None and tryInt(cat_id) > 0 else (m.category_id or None) + m['profile_id'] = params.get('profile_id', default_profile.get('id')) + m['category_id'] = tryInt(cat_id) if cat_id is not None and tryInt(cat_id) > 0 else (m['category_id'] or None) else: log.debug('Movie already exists, not updating: %s', params) added = False if force_readd: - m.status_id = status_id if status_id else status_active.get('id') - m.last_edit = int(time.time()) + m['status'] = status_id if status_id else 'active' + m['last_edit'] = int(time.time()) do_search = True - db.commit() + db.update(m) # Remove releases - available_status = fireEvent('status.get', 'available', single = True) - for rel in m.releases: - if rel.status_id is available_status.get('id'): + for rel in db.run('release', 'for_media', m['_id']): + if rel['status'] is 'available': db.delete(rel) - db.commit() - movie_dict = m.to_dict(self.default_dict) + movie_dict = db.run('movie', 'to_dict', m['_id']) if do_search and search_after: - onComplete = self.createOnComplete(m.id) + onComplete = self.createOnComplete(m['_id']) onComplete() if added: if params.get('title'): message = 'Successfully added "%s" to your wanted list.' % params.get('title', '') else: - title = getTitle(m.library) + title = getTitle(m) if title: message = 'Successfully added "%s" to your wanted list.' % title else: @@ -142,10 +142,7 @@ class MovieBase(MovieTypeBase): return movie_dict except: - log.error('Failed deleting media: %s', traceback.format_exc()) - db.rollback() - finally: - db.close() + log.error('Failed adding media: %s', traceback.format_exc()) def addView(self, **kwargs): add_dict = self.add(params = kwargs) @@ -158,49 +155,44 @@ class MovieBase(MovieTypeBase): def edit(self, id = '', **kwargs): try: - db = get_session() - - available_status = fireEvent('status.get', 'available', single = True) + db = get_db() ids = splitString(id) for media_id in ids: - m = db.query(Media).filter_by(id = media_id).first() - if not m: - continue + try: + m = db.get('media', media_id) + m['profile_id'] = kwargs.get('profile_id') - m.profile_id = kwargs.get('profile_id') + cat_id = kwargs.get('category_id') + if cat_id is not None: + m['category_id'] = tryInt(cat_id) if tryInt(cat_id) > 0 else None - cat_id = kwargs.get('category_id') - if cat_id is not None: - m.category_id = tryInt(cat_id) if tryInt(cat_id) > 0 else None + # Remove releases + for rel in db.run('release', 'for_media', m['_id']): + if rel['status'] is 'available': + db.delete(rel) - # Remove releases - for rel in m.releases: - if rel.status_id is available_status.get('id'): - db.delete(rel) - db.commit() + # Default title + if kwargs.get('default_title'): + for title in m['titles']: + title.default = toUnicode(kwargs.get('default_title', '')).lower() == toUnicode(title.title).lower() - # Default title - if kwargs.get('default_title'): - for title in m.library.titles: - title.default = toUnicode(kwargs.get('default_title', '')).lower() == toUnicode(title.title).lower() + db.update(m) - db.commit() + fireEvent('media.restatus', m['_id']) - fireEvent('media.restatus', m.id) + movie_dict = db.run('media', 'to_dict', m['_id']) + fireEventAsync('movie.searcher.single', movie_dict, on_complete = self.createNotifyFront(media_id)) - movie_dict = m.to_dict(self.default_dict) - fireEventAsync('movie.searcher.single', movie_dict, on_complete = self.createNotifyFront(media_id)) + except: + log.error('Can\'t edit non-existing media') return { 'success': True, } except: - log.error('Failed deleting media: %s', traceback.format_exc()) - db.rollback() - finally: - db.close() + log.error('Failed editing media: %s', traceback.format_exc()) return { 'success': False, diff --git a/couchpotato/core/media/movie/library/movie/main.py b/couchpotato/core/media/movie/library/movie/main.py index 034a8fb0..663c6d04 100644 --- a/couchpotato/core/media/movie/library/movie/main.py +++ b/couchpotato/core/media/movie/library/movie/main.py @@ -62,7 +62,7 @@ class MovieLibraryPlugin(LibraryBase): log.error('Failed adding media: %s', traceback.format_exc()) db.rollback() finally: - db.close() + pass #db.close() return {} @@ -130,31 +130,24 @@ class MovieLibraryPlugin(LibraryBase): # Files images = info.get('images', []) + library['files'] = [] for image_type in ['poster']: for image in images.get(image_type, []): if not isinstance(image, (str, unicode)): continue file_path = fireEvent('file.download', url = image, single = True) - if file_path: - file_obj = fireEvent('file.add', path = file_path, type_tuple = ('image', image_type), single = True) - try: - file_obj = db.query(File).filter_by(id = file_obj.get('id')).one() - library.files.append(file_obj) - db.commit() - break - except: - log.debug('Failed to attach to library: %s', traceback.format_exc()) - db.rollback() + # TODO: save in movie doc + library['files'].append({ + 'type': 'image_%s' % image_type, + 'path': file_path + }) library_dict = library.to_dict(self.default_dict) return library_dict except: log.error('Failed update media: %s', traceback.format_exc()) - db.rollback() - finally: - db.close() return {} @@ -180,7 +173,7 @@ class MovieLibraryPlugin(LibraryBase): log.error('Failed updating release dates: %s', traceback.format_exc()) db.rollback() finally: - db.close() + pass #db.close() return {} diff --git a/couchpotato/core/media/movie/searcher/main.py b/couchpotato/core/media/movie/searcher/main.py index bde5b44b..b4665deb 100644 --- a/couchpotato/core/media/movie/searcher/main.py +++ b/couchpotato/core/media/movie/searcher/main.py @@ -1,4 +1,4 @@ -from couchpotato import get_session +from couchpotato import get_session, get_db from couchpotato.api import addApiView from couchpotato.core.event import addEvent, fireEvent, fireEventAsync from couchpotato.core.helpers.encoding import simplifyString @@ -51,7 +51,7 @@ class MovieSearcher(SearcherBase, MovieTypeBase): }) if self.conf('run_on_launch'): - addEvent('app.load', self.searchAll) + addEvent('app.load2', self.searchAll) def searchAllView(self, **kwargs): @@ -71,27 +71,16 @@ class MovieSearcher(SearcherBase, MovieTypeBase): self.in_progress = True fireEvent('notify.frontend', type = 'movie.searcher.started', data = True, message = 'Full search started') - db = get_session() + db = get_db() - movies_raw = db.query(Media).filter( - Media.status.has(identifier = 'active') - ).all() + movies = db.get_many('movie_status', 'active') - random.shuffle(movies_raw) - - movies = [] - for m in movies_raw: - movies.append(m.to_dict({ - 'category': {}, - 'profile': {'types': {'quality': {}}}, - 'releases': {'status': {}, 'quality': {}}, - 'library': {'titles': {}, 'files': {}}, - 'files': {}, - })) + #TODO: random.shuffle(movies_raw) + total = db.count(db.get_many, 'movie_status', 'active') self.in_progress = { - 'total': len(movies), - 'to_go': len(movies), + 'total': total, + 'to_go': total, } try: @@ -99,13 +88,15 @@ class MovieSearcher(SearcherBase, MovieTypeBase): for movie in movies: + movie_dict = db.run('media', 'to_dict', movie['_id']) + try: - self.single(movie, search_protocols) + self.single(movie_dict, search_protocols) except IndexError: - log.error('Forcing library update for %s, if you see this often, please report: %s', (movie['library']['identifier'], traceback.format_exc())) - fireEvent('library.update.movie', movie['library']['identifier']) + log.error('Forcing library update for %s, if you see this often, please report: %s', (movie['identifier'], traceback.format_exc())) + fireEvent('library.update.movie', movie['identifier']) except: - log.error('Search failed for %s: %s', (movie['library']['identifier'], traceback.format_exc())) + log.error('Search failed for %s: %s', (movie['identifier'], traceback.format_exc())) self.in_progress['to_go'] -= 1 @@ -117,7 +108,6 @@ class MovieSearcher(SearcherBase, MovieTypeBase): pass self.in_progress = False - db.close() def single(self, movie, search_protocols = None, manual = False): @@ -132,32 +122,30 @@ class MovieSearcher(SearcherBase, MovieTypeBase): except SearchSetupError: return - done_status = fireEvent('status.get', 'done', single = True) - - if not movie['profile'] or (movie['status_id'] == done_status.get('id') and not manual): + if not movie['profile_id'] or (movie['status'] == 'done' and not manual): log.debug('Movie doesn\'t have a profile or already done, assuming in manage tab.') return pre_releases = fireEvent('quality.pre_releases', single = True) release_dates = fireEvent('library.update.movie.release_date', identifier = movie['library']['identifier'], merge = True) - available_status, ignored_status, failed_status = fireEvent('status.get', ['available', 'ignored', 'failed'], single = True) found_releases = [] too_early_to_search = [] - default_title = getTitle(movie['library']) + default_title = getTitle(movie) if not default_title: log.error('No proper info found for movie, removing it from library to cause it from having more issues.') - fireEvent('media.delete', movie['id'], single = True) + fireEvent('media.delete', movie['_id'], single = True) return - fireEvent('notify.frontend', type = 'movie.searcher.started', data = {'id': movie['id']}, message = 'Searching for "%s"' % default_title) + fireEvent('notify.frontend', type = 'movie.searcher.started', data = {'id': movie['_id']}, message = 'Searching for "%s"' % default_title) - db = get_session() + db = get_db() ret = False + for quality_type in movie['profile']['types']: - if not self.conf('always_search') and not self.couldBeReleased(quality_type['quality']['identifier'] in pre_releases, release_dates, movie['library']['year']): + if not self.conf('always_search') and not self.couldBeReleased(quality_type['quality']['identifier'] in pre_releases, release_dates, movie['year']): too_early_to_search.append(quality_type['quality']['identifier']) continue @@ -165,7 +153,7 @@ class MovieSearcher(SearcherBase, MovieTypeBase): # See if better quality is available for release in movie['releases']: - if release['quality']['order'] <= quality_type['quality']['order'] and release['status_id'] not in [available_status.get('id'), ignored_status.get('id'), failed_status.get('id')]: + if release['quality']['order'] <= quality_type['quality']['order'] and release['status'] not in ['available', 'ignored', 'failed']: has_better_quality += 1 # Don't search for quality lower then already available. @@ -179,7 +167,7 @@ class MovieSearcher(SearcherBase, MovieTypeBase): log.debug('Nothing found for %s in %s', (default_title, quality_type['quality']['label'])) # Check if movie isn't deleted while searching - if not db.query(Media).filter_by(id = movie.get('id')).first(): + if not fireEvent('media.get', movie.get('_id'), single = True): break # Add them to this movie releases list @@ -191,12 +179,12 @@ class MovieSearcher(SearcherBase, MovieTypeBase): # Remove releases that aren't found anymore for release in movie.get('releases', []): - if release.get('status_id') == available_status.get('id') and release.get('identifier') not in found_releases: - fireEvent('release.delete', release.get('id'), single = True) + if release.get('status') == 'available' and release.get('identifier') not in found_releases: + fireEvent('release.delete', release.get('_id'), single = True) else: log.info('Better quality (%s) already available or snatched for %s', (quality_type['quality']['label'], default_title)) - fireEvent('media.restatus', movie['id']) + fireEvent('media.restatus', movie['_id']) break # Break if CP wants to shut down @@ -206,9 +194,9 @@ class MovieSearcher(SearcherBase, MovieTypeBase): if len(too_early_to_search) > 0: log.info2('Too early to search for %s, %s', (too_early_to_search, default_title)) - fireEvent('notify.frontend', type = 'movie.searcher.ended', data = {'id': movie['id']}) + fireEvent('notify.frontend', type = 'movie.searcher.ended', data = {'id': movie['_id']}) - db.close() + pass #db.close() return ret def correctRelease(self, nzb = None, media = None, quality = None, **kwargs): @@ -329,31 +317,23 @@ class MovieSearcher(SearcherBase, MovieTypeBase): def tryNextRelease(self, media_id, manual = False): - snatched_status, done_status, ignored_status = fireEvent('status.get', ['snatched', 'done', 'ignored'], single = True) - try: - db = get_session() - rels = db.query(Release) \ - .filter_by(movie_id = media_id) \ - .filter(Release.status_id.in_([snatched_status.get('id'), done_status.get('id')])) \ - .all() + db = get_db() + rels = db.run('media', 'with_status', media_id, status = ['snatched', 'done']) for rel in rels: - rel.status_id = ignored_status.get('id') - db.commit() + rel['status'] = 'ignored' + db.update(rel) - movie_dict = fireEvent('media.get', media_id = media_id, single = True) - log.info('Trying next release for: %s', getTitle(movie_dict['library'])) - fireEvent('movie.searcher.single', movie_dict, manual = manual) + movie_dict = db.run('media', 'to_dict', media_id) + log.info('Trying next release for: %s', getTitle(movie_dict)) + self.single(movie_dict, manual = manual) return True except: log.error('Failed searching for next release: %s', traceback.format_exc()) - db.rollback() return False - finally: - db.close() def getSearchTitle(self, media): if media['type'] == 'movie': diff --git a/couchpotato/core/media/movie/suggestion/main.py b/couchpotato/core/media/movie/suggestion/main.py index d1d64089..eaccbe64 100644 --- a/couchpotato/core/media/movie/suggestion/main.py +++ b/couchpotato/core/media/movie/suggestion/main.py @@ -33,7 +33,7 @@ class Suggestion(Plugin): .options(joinedload_all('library')) \ .filter(or_(*[Media.status.has(identifier = s) for s in ['active', 'done']])).all() movies = [x.library.identifier for x in active_movies] - db.close() + pass #db.close() if not ignored or len(ignored) == 0: ignored = splitString(Env.prop('suggest_ignore', default = '')) @@ -98,7 +98,7 @@ class Suggestion(Plugin): .filter(Media.status_id.in_([active_status.get('id'), done_status.get('id')])).all() movies = [x[0] for x in active_movies] movies.extend(seen) - db.close() + pass #db.close() ignored.extend([x.get('imdb') for x in cached_suggestion]) suggestions = fireEvent('movie.suggest', movies = movies, ignore = removeDuplicate(ignored), single = True) diff --git a/couchpotato/core/notifications/core/index.py b/couchpotato/core/notifications/core/index.py new file mode 100644 index 00000000..e51124d6 --- /dev/null +++ b/couchpotato/core/notifications/core/index.py @@ -0,0 +1,42 @@ +import time +from CodernityDB.tree_index import TreeBasedIndex + + +class NotificationIndex(TreeBasedIndex): + + custom_header = """from CodernityDB.tree_index import TreeBasedIndex +import time""" + + def __init__(self, *args, **kwargs): + kwargs['key_format'] = 'I' + super(NotificationIndex, self).__init__(*args, **kwargs) + + def make_key(self, key): + return key + + def make_key_value(self, data): + if data.get('type') == 'notification': + added = data.get('added', time.time()) + data['added'] = added + + return added, None + + +class NotificationUnreadIndex(TreeBasedIndex): + + custom_header = """from CodernityDB.tree_index import TreeBasedIndex +import time""" + + def __init__(self, *args, **kwargs): + kwargs['key_format'] = 'I' + super(NotificationUnreadIndex, self).__init__(*args, **kwargs) + + def make_key(self, key): + return key + + def make_key_value(self, data): + if data.get('type') == 'notification' and not data.get('read'): + added = data.get('added', time.time()) + data['added'] = added + + return added, None diff --git a/couchpotato/core/notifications/core/main.py b/couchpotato/core/notifications/core/main.py index d77758b6..4a245cdd 100644 --- a/couchpotato/core/notifications/core/main.py +++ b/couchpotato/core/notifications/core/main.py @@ -1,14 +1,13 @@ -from couchpotato import get_session +from couchpotato import get_db from couchpotato.api import addApiView, addNonBlockApiView from couchpotato.core.event import addEvent, fireEvent from couchpotato.core.helpers.encoding import toUnicode from couchpotato.core.helpers.variable import tryInt, splitString from couchpotato.core.logger import CPLog from couchpotato.core.notifications.base import Notification -from couchpotato.core.settings.model import Notification as Notif +from .index import NotificationIndex, NotificationUnreadIndex from couchpotato.environment import Env from operator import itemgetter -from sqlalchemy.sql.expression import or_ import threading import time import traceback @@ -58,48 +57,55 @@ class CoreNotifier(Notification): fireEvent('schedule.interval', 'core.check_messages', self.checkMessages, hours = 12, single = True) fireEvent('schedule.interval', 'core.clean_messages', self.cleanMessages, seconds = 15, single = True) - addEvent('app.load', self.clean) - addEvent('app.load', self.checkMessages) + addEvent('app.load2', self.clean) + addEvent('app.load2', self.checkMessages) + + addEvent('database.setup', self.databaseSetup) self.messages = [] self.listeners = [] self.m_lock = threading.Lock() - def clean(self): + def databaseSetup(self): + + db = get_db() try: - db = get_session() - db.query(Notif).filter(Notif.added <= (int(time.time()) - 2419200)).delete() - db.commit() + db.add_index(NotificationIndex(db.path, 'notification')) + except: + log.debug('Index already exists') + db.edit_index(NotificationIndex(db.path, 'notification')) + + try: + db.add_index(NotificationUnreadIndex(db.path, 'notification_unread')) + except: + log.debug('Index already exists') + db.edit_index(NotificationUnreadIndex(db.path, 'notification_unread')) + + def clean(self): + try: + db = get_db() + for n in db.all('notification', with_doc = True): + if n['doc']['added'] <= (int(time.time()) - 2419200): + db.delete(n['doc']) except: log.error('Failed cleaning notification: %s', traceback.format_exc()) - db.rollback() - finally: - db.close() def markAsRead(self, ids = None, **kwargs): ids = splitString(ids) if ids else None try: - db = get_session() - - if ids: - q = db.query(Notif).filter(or_(*[Notif.id == tryInt(s) for s in ids])) - else: - q = db.query(Notif).filter_by(read = False) - - q.update({Notif.read: True}) - db.commit() - + db = get_db() + for x in db.all('notification_unread', with_doc = True): + if not ids or x['_id'] in ids: + x['doc']['read'] = True + db.update(x['doc']) return { 'success': True } except: log.error('Failed mark as read: %s', traceback.format_exc()) - db.rollback() - finally: - db.close() return { 'success': False @@ -107,26 +113,20 @@ class CoreNotifier(Notification): def listView(self, limit_offset = None, **kwargs): - db = get_session() - - q = db.query(Notif) + db = get_db() if limit_offset: splt = splitString(limit_offset) limit = splt[0] offset = 0 if len(splt) is 1 else splt[1] - q = q.limit(limit).offset(offset) + results = db.get_many('notification', limit = limit, offset = offset, with_doc = True) else: - q = q.limit(200) + results = db.get_many('notification', limit = 200, with_doc = True) - results = q.all() notifications = [] for n in results: - ndict = n.to_dict() - ndict['type'] = 'notification' - notifications.append(ndict) + notifications.append(n['doc']) - db.close() return { 'success': True, 'empty': len(notifications) == 0, @@ -156,30 +156,23 @@ class CoreNotifier(Notification): if not data: data = {} try: - db = get_session() + db = get_db() data['notification_type'] = listener if listener else 'unknown' - n = Notif( - message = toUnicode(message), - data = data - ) - db.add(n) - db.commit() + n = { + 'type': 'notification', + 'time': time.time(), + 'message': toUnicode(message), + 'data': data + } + db.insert(n) - ndict = n.to_dict() - ndict['type'] = 'notification' - ndict['time'] = time.time() + self.frontend(type = listener, data = n) - self.frontend(type = listener, data = data) - - db.close() return True except: log.error('Failed notify: %s', traceback.format_exc()) - db.rollback() - finally: - db.close() def frontend(self, type = 'notification', data = None, message = None): if not data: data = {} @@ -278,18 +271,13 @@ class CoreNotifier(Notification): # Get unread if init: - db = get_session() + db = get_db() - notifications = db.query(Notif) \ - .filter(or_(Notif.read == False, Notif.added > (time.time() - 259200))) \ - .all() + notifications = db.all('notification_unread', with_doc = True) for n in notifications: - ndict = n.to_dict() - ndict['type'] = 'notification' - messages.append(ndict) - - db.close() + if n['doc'].get('added') > (time.time() - 259200): + messages.append(n['doc']) return { 'success': True, diff --git a/couchpotato/core/notifications/growl/main.py b/couchpotato/core/notifications/growl/main.py index a3927ed2..68d73cb1 100644 --- a/couchpotato/core/notifications/growl/main.py +++ b/couchpotato/core/notifications/growl/main.py @@ -16,7 +16,7 @@ class Growl(Notification): super(Growl, self).__init__() if self.isEnabled(): - addEvent('app.load', self.register) + addEvent('app.load2', self.register) def register(self): if self.registered: return diff --git a/couchpotato/core/plugins/automation/main.py b/couchpotato/core/plugins/automation/main.py index 2edcd3be..44d7673f 100644 --- a/couchpotato/core/plugins/automation/main.py +++ b/couchpotato/core/plugins/automation/main.py @@ -10,10 +10,10 @@ class Automation(Plugin): def __init__(self): - addEvent('app.load', self.setCrons) + addEvent('app.load2', self.setCrons) if not Env.get('dev'): - addEvent('app.load', self.addMovies) + addEvent('app.load2', self.addMovies) addEvent('setting.save.automation.hour.after', self.setCrons) diff --git a/couchpotato/core/plugins/category/main.py b/couchpotato/core/plugins/category/main.py index 41965b03..b26a8293 100644 --- a/couchpotato/core/plugins/category/main.py +++ b/couchpotato/core/plugins/category/main.py @@ -42,7 +42,7 @@ class CategoryPlugin(Plugin): for category in categories: temp.append(category.to_dict()) - db.close() + pass #db.close() return temp def save(self, **kwargs): @@ -66,7 +66,7 @@ class CategoryPlugin(Plugin): category_dict = c.to_dict() - db.close() + pass #db.close() return { 'success': True, 'category': category_dict @@ -75,7 +75,7 @@ class CategoryPlugin(Plugin): log.error('Failed: %s', traceback.format_exc()) db.rollback() finally: - db.close() + pass #db.close() return { 'success': False, @@ -96,7 +96,7 @@ class CategoryPlugin(Plugin): db.commit() - db.close() + pass #db.close() return { 'success': True } @@ -104,7 +104,7 @@ class CategoryPlugin(Plugin): log.error('Failed: %s', traceback.format_exc()) db.rollback() finally: - db.close() + pass #db.close() return { 'success': False @@ -129,7 +129,7 @@ class CategoryPlugin(Plugin): except Exception as e: message = log.error('Failed deleting category: %s', e) - db.close() + pass #db.close() return { 'success': success, 'message': message @@ -138,7 +138,7 @@ class CategoryPlugin(Plugin): log.error('Failed: %s', traceback.format_exc()) db.rollback() finally: - db.close() + pass #db.close() return { 'success': False @@ -158,4 +158,4 @@ class CategoryPlugin(Plugin): log.error('Failed: %s', traceback.format_exc()) db.rollback() finally: - db.close() + pass #db.close() diff --git a/couchpotato/core/plugins/custom/main.py b/couchpotato/core/plugins/custom/main.py index a15c915c..42400994 100644 --- a/couchpotato/core/plugins/custom/main.py +++ b/couchpotato/core/plugins/custom/main.py @@ -10,7 +10,7 @@ log = CPLog(__name__) class Custom(Plugin): def __init__(self): - addEvent('app.load', self.createStructure) + addEvent('app.load2', self.createStructure) def createStructure(self): diff --git a/couchpotato/core/plugins/dashboard/main.py b/couchpotato/core/plugins/dashboard/main.py index 73866411..bc14f4b5 100644 --- a/couchpotato/core/plugins/dashboard/main.py +++ b/couchpotato/core/plugins/dashboard/main.py @@ -119,7 +119,7 @@ class Dashboard(Plugin): 'files': {}, })) - db.close() + pass #db.close() return { 'success': True, 'empty': len(movies) == 0, diff --git a/couchpotato/core/plugins/file/main.py b/couchpotato/core/plugins/file/main.py index 18071547..c24d0d58 100644 --- a/couchpotato/core/plugins/file/main.py +++ b/couchpotato/core/plugins/file/main.py @@ -19,9 +19,7 @@ log = CPLog(__name__) class FileManager(Plugin): def __init__(self): - addEvent('file.add', self.add) addEvent('file.download', self.download) - addEvent('file.types', self.getTypes) addApiView('file.cache/(.*)', self.showCacheFile, static = True, docs = { 'desc': 'Return a file from the cp_data/cache directory', @@ -31,36 +29,8 @@ class FileManager(Plugin): 'return': {'type': 'file'} }) - addApiView('file.types', self.getTypesView, docs = { - 'desc': 'Return a list of all the file types and their ids.', - 'return': {'type': 'object', 'example': """{ - 'types': [ - { - "identifier": "poster_original", - "type": "image", - "id": 1, - "name": "Poster_original" - }, - { - "identifier": "poster", - "type": "image", - "id": 2, - "name": "Poster" - }, - etc - ] -}"""} - }) - - addEvent('app.load', self.cleanup) - addEvent('app.load', self.init) - - def init(self): - - for type_tuple in Scanner.file_types.values(): - self.getType(type_tuple) - def cleanup(self): + # TODO: unused # Wait a bit after starting before cleanup time.sleep(3) @@ -77,9 +47,6 @@ class FileManager(Plugin): os.remove(file_path) except: log.error('Failed removing unused file: %s', traceback.format_exc()) - db.rollback() - finally: - db.close() def showCacheFile(self, route, **kwargs): Env.get('app').add_handlers(".*$", [('%s%s' % (Env.get('api_base'), route), StaticFileHandler, {'path': Env.get('cache_dir')})]) @@ -101,76 +68,3 @@ class FileManager(Plugin): self.createFile(dest, filedata, binary = True) return dest - - def add(self, path = '', part = 1, type_tuple = (), available = 1, properties = None): - if not properties: properties = {} - - try: - db = get_session() - type_id = self.getType(type_tuple).get('id') - - f = db.query(File).filter(File.path == toUnicode(path)).first() - if not f: - f = File() - db.add(f) - - f.path = toUnicode(path) - f.part = part - f.available = available - f.type_id = type_id - - db.commit() - - file_dict = f.to_dict() - - return file_dict - except: - log.error('Failed adding file: %s, %s', (path, traceback.format_exc())) - db.rollback() - finally: - db.close() - - def getType(self, type_tuple): - - try: - db = get_session() - type_type, type_identifier = type_tuple - - ft = db.query(FileType).filter_by(identifier = type_identifier).first() - if not ft: - ft = FileType( - type = toUnicode(type_type), - identifier = type_identifier, - name = toUnicode(type_identifier[0].capitalize() + type_identifier[1:]) - ) - db.add(ft) - db.commit() - - type_dict = ft.to_dict() - - return type_dict - except: - log.error('Failed getting type: %s, %s', (type_tuple, traceback.format_exc())) - db.rollback() - finally: - db.close() - - - def getTypes(self): - - db = get_session() - - results = db.query(FileType).all() - - types = [] - for type_object in results: - types.append(type_object.to_dict()) - - db.close() - return types - - def getTypesView(self, **kwargs): - - return { - 'types': self.getTypes() - } diff --git a/couchpotato/core/plugins/manage/main.py b/couchpotato/core/plugins/manage/main.py index 2f297491..47081692 100644 --- a/couchpotato/core/plugins/manage/main.py +++ b/couchpotato/core/plugins/manage/main.py @@ -47,7 +47,7 @@ class Manage(Plugin): }) if not Env.get('dev') and self.conf('startup_scan'): - addEvent('app.load', self.updateLibraryQuick) + addEvent('app.load2', self.updateLibraryQuick) def getProgress(self, **kwargs): return { diff --git a/couchpotato/core/plugins/nosql/__init__.py b/couchpotato/core/plugins/nosql/__init__.py new file mode 100644 index 00000000..7f0e952e --- /dev/null +++ b/couchpotato/core/plugins/nosql/__init__.py @@ -0,0 +1,7 @@ +from .main import NoSQL + + +def start(): + return NoSQL() + +config = [] diff --git a/couchpotato/core/plugins/nosql/index.py b/couchpotato/core/plugins/nosql/index.py new file mode 100644 index 00000000..e1361799 --- /dev/null +++ b/couchpotato/core/plugins/nosql/index.py @@ -0,0 +1,20 @@ +from hashlib import md5 +from CodernityDB.tree_index import TreeBasedIndex + + +class ReleaseIndex(TreeBasedIndex): + + def __init__(self, *args, **kwargs): + kwargs['key_format'] = '16s' + super(ReleaseIndex, self).__init__(*args, **kwargs) + + def make_key(self, key): + return md5(key).digest() + + def make_key_value(self, data): + if data.get('type') == 'release': + return md5(data['media_id']).digest(), None + + def run_for_media(self, db, media_id): + for release in db.get_many('release', media_id, with_doc=True): + yield release['doc'] diff --git a/couchpotato/core/plugins/nosql/main.py b/couchpotato/core/plugins/nosql/main.py new file mode 100644 index 00000000..11f15cb9 --- /dev/null +++ b/couchpotato/core/plugins/nosql/main.py @@ -0,0 +1,72 @@ +import time +from couchpotato import CPLog +from couchpotato.core.plugins.base import Plugin +from .index import ReleaseIndex, MediaIMDBIndex, TitleIndex + +log = CPLog(__name__) + + +class NoSQL(Plugin): + + db = None + + def test(self): + + db = self.db + + try: db.add_index(ReleaseIndex(db.path, 'release')) + except: log.debug('Index already exists') + + for id in range(10): + media = db.insert({ + 'type': 'media', + 'tmdb': id, + 'imdb': 'tt%s' % id, + 'last_edit': 0, + 'status': 'active', + 'title': 'Lord of the Rings: The Return of the King', + 'year': 2011, + 'profile_id': 0, + 'category_id': 0, + }) + + db.insert({ + 'media_id': media['_id'], + 'type': 'title', + 'title': 'Lord of the Rings: The Return of the King', + }) + + for x in range(40): + db.insert({ + 'media_id': media['_id'], + 'type': 'release', + 'name': 'Release %s' % x + }) + + print db.count(db.all, 'media') + + m = db.get('media', 'tt0') + db.get('id', m['_id']) + + start = time.time() + print list(db.get_many('media_title', 'lord of')) + print time.time() - start + + return + + return + + for media in db.all('media', with_doc = True): + doc = media['doc'] + for r in db.run('release', 'for_media', media['_id']): + db.delete(r) + + db.delete(doc) + break + + start = time.time() + db.reindex() + print time.time() - start + + print db.count(db.all, 'media') + print db.count(db.all, 'release') diff --git a/couchpotato/core/plugins/profile/main.py b/couchpotato/core/plugins/profile/main.py index cc048733..9d721852 100644 --- a/couchpotato/core/plugins/profile/main.py +++ b/couchpotato/core/plugins/profile/main.py @@ -1,5 +1,5 @@ import traceback -from couchpotato import get_session +from couchpotato import get_session, get_db from couchpotato.api import addApiView from couchpotato.core.event import addEvent, fireEvent from couchpotato.core.helpers.encoding import toUnicode @@ -31,27 +31,24 @@ class ProfilePlugin(Plugin): }) addEvent('app.initialize', self.fill, priority = 90) - addEvent('app.load', self.forceDefaults) + addEvent('app.load2', self.forceDefaults) def forceDefaults(self): # Get all active movies without profile - active_status = fireEvent('status.get', 'active', single = True) - try: - db = get_session() - movies = db.query(Media).filter(Media.status_id == active_status.get('id'), Media.profile == None).all() + db = get_db() + medias = db.run('media', 'with_status', ['active']) - if len(movies) > 0: - default_profile = self.default() - for movie in movies: - movie.profile_id = default_profile.get('id') - db.commit() + profile_ids = [x.get('_id') for x in self.all()] + + for media in medias: + if media['profile_id'] not in profile_ids: + default_profile = self.default() + media['profile_id'] = default_profile.get('id') + db.update(media) except: log.error('Failed: %s', traceback.format_exc()) - db.rollback() - finally: - db.close() def allView(self, **kwargs): @@ -62,17 +59,10 @@ class ProfilePlugin(Plugin): def all(self): - db = get_session() - profiles = db.query(Profile) \ - .options(joinedload_all('types')) \ - .all() + db = get_db() + profiles = db.all('profile', with_doc = True) - temp = [] - for profile in profiles: - temp.append(profile.to_dict(self.to_dict)) - - db.close() - return temp + return list(profiles) def save(self, **kwargs): @@ -115,7 +105,7 @@ class ProfilePlugin(Plugin): log.error('Failed: %s', traceback.format_exc()) db.rollback() finally: - db.close() + pass #db.close() return { 'success': False @@ -128,7 +118,7 @@ class ProfilePlugin(Plugin): .options(joinedload_all('types')) \ .first() default_dict = default.to_dict(self.to_dict) - db.close() + pass #db.close() return default_dict @@ -154,7 +144,7 @@ class ProfilePlugin(Plugin): log.error('Failed: %s', traceback.format_exc()) db.rollback() finally: - db.close() + pass #db.close() return { 'success': False @@ -188,7 +178,7 @@ class ProfilePlugin(Plugin): log.error('Failed: %s', traceback.format_exc()) db.rollback() finally: - db.close() + pass #db.close() return { 'success': False @@ -243,6 +233,6 @@ class ProfilePlugin(Plugin): log.error('Failed: %s', traceback.format_exc()) db.rollback() finally: - db.close() + pass #db.close() return False diff --git a/couchpotato/core/plugins/quality/main.py b/couchpotato/core/plugins/quality/main.py index cc6215d3..e5070093 100644 --- a/couchpotato/core/plugins/quality/main.py +++ b/couchpotato/core/plugins/quality/main.py @@ -79,7 +79,7 @@ class QualityPlugin(Plugin): self.cached_qualities = temp - db.close() + pass #db.close() return temp def single(self, identifier = ''): @@ -91,7 +91,7 @@ class QualityPlugin(Plugin): if quality: quality_dict = dict(self.getQuality(quality.identifier), **quality.to_dict()) - db.close() + pass #db.close() return quality_dict def getQuality(self, identifier): @@ -119,7 +119,7 @@ class QualityPlugin(Plugin): log.error('Failed: %s', traceback.format_exc()) db.rollback() finally: - db.close() + pass #db.close() return { 'success': False @@ -181,7 +181,7 @@ class QualityPlugin(Plugin): log.error('Failed: %s', traceback.format_exc()) db.rollback() finally: - db.close() + pass #db.close() return False diff --git a/couchpotato/core/plugins/release/main.py b/couchpotato/core/plugins/release/main.py index 1e1352c5..9066fa18 100644 --- a/couchpotato/core/plugins/release/main.py +++ b/couchpotato/core/plugins/release/main.py @@ -1,4 +1,4 @@ -from couchpotato import get_session, md5 +from couchpotato import get_session, md5, get_db from couchpotato.api import addApiView from couchpotato.core.event import fireEvent, addEvent from couchpotato.core.helpers.encoding import ss, toUnicode @@ -72,34 +72,32 @@ class Release(Plugin): done_status, available_status, snatched_status, downloaded_status, ignored_status = \ fireEvent('status.get', ['done', 'available', 'snatched', 'downloaded', 'ignored'], single = True) - db = get_session() + db = get_db() # get movies last_edit more than a week ago - media = db.query(Media) \ - .filter(Media.status_id == done_status.get('id'), Media.last_edit < (now - week)) \ - .all() + medias = db.run('media', 'with_status', ['done']) - for item in media: - for rel in item.releases: + for media in medias: + if media['last_edit'] > (now - week): + continue + + for rel in db.run('release', 'for_media', media['_id']): # Remove all available releases - if rel.status_id in [available_status.get('id')]: - fireEvent('release.delete', id = rel.id, single = True) - # Set all snatched and downloaded releases to ignored to make sure they are ignored when re-adding the move - elif rel.status_id in [snatched_status.get('id'), downloaded_status.get('id')]: - self.updateStatus(id = rel.id, status = ignored_status) + if rel['status'] in ['available']: + fireEvent('release.delete', id = rel['_id'], single = True) - db.close() + # Set all snatched and downloaded releases to ignored to make sure they are ignored when re-adding the move + elif rel['status'] in ['snatched', 'downloaded']: + self.updateStatus(id = rel['id'], status = ignored_status) def add(self, group): try: - db = get_session() + db = get_db() identifier = '%s.%s.%s' % (group['library']['identifier'], group['meta_data'].get('audio', 'unknown'), group['meta_data']['quality']['identifier']) - done_status, snatched_status = fireEvent('status.get', ['done', 'snatched'], single = True) - # Add movie media = db.query(Media).filter_by(library_id = group['library'].get('id')).first() if not media: @@ -129,41 +127,28 @@ class Release(Plugin): db.commit() # Add each file type - added_files = [] + rel['files'] = [] for type in group['files']: for cur_file in group['files'][type]: - added_file = self.saveFile(cur_file, type = type, include_media_info = type is 'movie') - added_files.append(added_file.get('id')) + added_file = self.saveFile(cur_file, type = type) + rel['files'].append(added_file.get('id')) - # Add the release files in batch - try: - added_files = db.query(File).filter(or_(*[File.id == x for x in added_files])).all() - rel.files.extend(added_files) - db.commit() - except: - log.debug('Failed to attach "%s" to release: %s', (added_files, traceback.format_exc())) - - fireEvent('media.restatus', media.id) + fireEvent('media.restatus', media['_id']) return True except: log.error('Failed: %s', traceback.format_exc()) - db.rollback() - finally: - db.close() return False def saveFile(self, filepath, type = 'unknown', include_media_info = False): - properties = {} - - # Get media info for files - if include_media_info: - properties = {} - # Check database and update/insert if necessary - return fireEvent('file.add', path = filepath, part = fireEvent('scanner.partnumber', file, single = True), type_tuple = Scanner.file_types.get(type), properties = properties, single = True) + return { + 'type': '%s_%s' % Scanner.file_types.get(type), + 'path': filepath, + 'part': fireEvent('scanner.partnumber', file, single = True), + } def deleteView(self, id = None, **kwargs): @@ -171,60 +156,58 @@ class Release(Plugin): 'success': self.delete(id) } - def delete(self, id): + def delete(self, release_id): try: - db = get_session() + db = get_db() - rel = db.query(Relea).filter_by(id = id).first() - if rel: - db.delete(rel) - db.commit() - return True + rel = db.get('release', release_id, with_doc = True) + db.delete(rel['doc']) + return True except: log.error('Failed: %s', traceback.format_exc()) - db.rollback() - finally: - db.close() return False - def clean(self, id): + def clean(self, release_id): try: - db = get_session() + db = get_db() - rel = db.query(Relea).filter_by(id = id).first() - if rel: - for release_file in rel.files: - if not os.path.isfile(ss(release_file.path)): - db.delete(release_file) - db.commit() + rel = db.get('release', release_id, with_doc = True) + files = [] + for release_file in rel['files']: + if os.path.isfile(ss(release_file['path'])): + files.append(release_file) - if len(rel.files) == 0: - self.delete(id) + if len(rel['files']) == 0: + self.delete(rel['_id']) + else: + rel['files'] = files + db.update(rel) - return True + return True except: log.error('Failed: %s', traceback.format_exc()) - db.rollback() - finally: - db.close() return False - def ignore(self, id = None, **kwargs): + def ignore(self, release_id = None, **kwargs): - db = get_session() + db = get_db() - rel = db.query(Relea).filter_by(id = id).first() - if rel: - ignored_status, failed_status, available_status = fireEvent('status.get', ['ignored', 'failed', 'available'], single = True) - self.updateStatus(id, available_status if rel.status_id in [ignored_status.get('id'), failed_status.get('id')] else ignored_status) + try: + rel = db.get('release', release_id, with_doc = True)['doc'] + self.updateStatus(release_id, 'available' if rel['status'] in ['ignored', 'failed'] else 'ignored') + + return { + 'success': True + } + except: + log.error('Failed: %s', traceback.format_exc()) - db.close() return { - 'success': True + 'success': False } def manualDownload(self, id = None, **kwargs): @@ -265,7 +248,7 @@ class Release(Plugin): if success: fireEvent('notify.frontend', type = 'release.manual_download', data = True, message = 'Successfully snatched "%s"' % item['name']) - db.close() + pass #db.close() return { 'success': success == True } @@ -361,7 +344,7 @@ class Release(Plugin): db.rollback() return False finally: - db.close() + pass #db.close() return True @@ -441,7 +424,7 @@ class Release(Plugin): log.error('Failed: %s', traceback.format_exc()) db.rollback() finally: - db.close() + pass #db.close() return [] @@ -458,7 +441,7 @@ class Release(Plugin): releases = [r.to_dict({'info': {}, 'files': {}}) for r in releases_raw] releases = sorted(releases, key = lambda k: k['info'].get('score', 0), reverse = True) - db.close() + pass #db.close() return releases def forMovieView(self, id = None, **kwargs): @@ -506,6 +489,6 @@ class Release(Plugin): log.error('Failed: %s', traceback.format_exc()) db.rollback() finally: - db.close() + pass #db.close() return False diff --git a/couchpotato/core/plugins/renamer/main.py b/couchpotato/core/plugins/renamer/main.py index c4c67deb..868dbc59 100755 --- a/couchpotato/core/plugins/renamer/main.py +++ b/couchpotato/core/plugins/renamer/main.py @@ -598,7 +598,7 @@ class Renamer(Plugin): break self.renaming_started = False - db.close() + pass #db.close() def getRenameExtras(self, extra_type = '', replacements = None, folder_name = '', file_name = '', destination = '', group = None, current_file = '', remove_multiple = False): if not group: group = {} @@ -1045,7 +1045,7 @@ Remove it if you want it to be renamed (again, or at least let it try again) self.checking_snatched = False return True finally: - db.close() + pass #db.close() def extendReleaseDownload(self, release_download): @@ -1068,7 +1068,7 @@ Remove it if you want it to be renamed (again, or at least let it try again) if not rls: log.error('Download ID %s from downloader %s not found in releases', (release_download.get('id'), release_download.get('downloader'))) - db.close() + pass #db.close() if rls: diff --git a/couchpotato/core/plugins/scanner/main.py b/couchpotato/core/plugins/scanner/main.py index 00a5b752..7e8fb9bb 100644 --- a/couchpotato/core/plugins/scanner/main.py +++ b/couchpotato/core/plugins/scanner/main.py @@ -441,7 +441,7 @@ class Scanner(Plugin): else: log.debug('Found no movies in the folder %s', folder) - db.close() + pass #db.close() return processed_movies def getMetaData(self, group, folder = '', release_download = None): @@ -609,7 +609,7 @@ class Scanner(Plugin): break except: pass - db.close() + pass #db.close() # Search based on identifiers if not imdb_id: diff --git a/couchpotato/core/plugins/status/__init__.py b/couchpotato/core/plugins/status/__init__.py deleted file mode 100644 index 204fbee7..00000000 --- a/couchpotato/core/plugins/status/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -from .main import StatusPlugin - - -def start(): - return StatusPlugin() - -config = [] diff --git a/couchpotato/core/plugins/status/main.py b/couchpotato/core/plugins/status/main.py deleted file mode 100644 index 55852610..00000000 --- a/couchpotato/core/plugins/status/main.py +++ /dev/null @@ -1,137 +0,0 @@ -import traceback -from couchpotato import get_session -from couchpotato.api import addApiView -from couchpotato.core.event import addEvent -from couchpotato.core.helpers.encoding import toUnicode -from couchpotato.core.logger import CPLog -from couchpotato.core.plugins.base import Plugin -from couchpotato.core.settings.model import Status - -log = CPLog(__name__) - - -class StatusPlugin(Plugin): - - statuses = { - 'needs_update': 'Needs update', - 'active': 'Active', - 'done': 'Done', - 'downloaded': 'Downloaded', - 'wanted': 'Wanted', - 'snatched': 'Snatched', - 'failed': 'Failed', - 'deleted': 'Deleted', - 'ignored': 'Ignored', - 'available': 'Available', - 'suggest': 'Suggest', - 'seeding': 'Seeding', - 'missing': 'Missing', - } - status_cached = {} - - def __init__(self): - addEvent('status.get', self.get) - addEvent('status.get_by_id', self.getById) - addEvent('status.all', self.all) - addEvent('app.initialize', self.fill) - addEvent('app.load', self.all) # Cache all statuses - - addApiView('status.list', self.list, docs = { - 'desc': 'Check for available update', - 'return': {'type': 'object', 'example': """{ - 'success': True, - 'list': array, statuses -}"""} - }) - - def list(self, **kwargs): - - return { - 'success': True, - 'list': self.all() - } - - def getById(self, id): - db = get_session() - status = db.query(Status).filter_by(id = id).first() - status_dict = status.to_dict() - db.close() - - return status_dict - - def all(self): - - db = get_session() - - statuses = db.query(Status).all() - - temp = [] - for status in statuses: - s = status.to_dict() - temp.append(s) - - # Update cache - self.status_cached[status.identifier] = s - - db.close() - return temp - - def get(self, identifiers): - - if not isinstance(identifiers, list): - identifiers = [identifiers] - - try: - db = get_session() - return_list = [] - - for identifier in identifiers: - - if self.status_cached.get(identifier): - return_list.append(self.status_cached.get(identifier)) - continue - - s = db.query(Status).filter_by(identifier = identifier).first() - if not s: - s = Status( - identifier = identifier, - label = toUnicode(identifier.capitalize()) - ) - db.add(s) - db.commit() - - status_dict = s.to_dict() - - self.status_cached[identifier] = status_dict - return_list.append(status_dict) - - return return_list if len(identifiers) > 1 else return_list[0] - except: - log.error('Failed: %s', traceback.format_exc()) - db.rollback() - finally: - db.close() - - def fill(self): - - try: - db = get_session() - - for identifier, label in self.statuses.items(): - s = db.query(Status).filter_by(identifier = identifier).first() - if not s: - log.info('Creating status: %s', label) - s = Status( - identifier = identifier, - label = toUnicode(label) - ) - db.add(s) - - s.label = toUnicode(label) - db.commit() - except: - log.error('Failed: %s', traceback.format_exc()) - db.rollback() - finally: - db.close() - diff --git a/couchpotato/core/plugins/status/static/status.js b/couchpotato/core/plugins/status/static/status.js deleted file mode 100644 index 2b8d30f3..00000000 --- a/couchpotato/core/plugins/status/static/status.js +++ /dev/null @@ -1,17 +0,0 @@ -var StatusBase = new Class({ - - setup: function(statuses){ - var self = this; - - self.statuses = statuses; - - }, - - get: function(id){ - return this.statuses.filter(function(status){ - return status.id == id - }).pick() - }, - -}); -window.Status = new StatusBase(); diff --git a/couchpotato/core/plugins/subtitle/main.py b/couchpotato/core/plugins/subtitle/main.py index e51df35e..b8d9a06b 100644 --- a/couchpotato/core/plugins/subtitle/main.py +++ b/couchpotato/core/plugins/subtitle/main.py @@ -41,7 +41,7 @@ class Subtitle(Plugin): # get subtitles for those files subliminal.list_subtitles(files, cache_dir = Env.get('cache_dir'), multi = True, languages = self.getLanguages(), services = self.services) - db.close() + pass #db.close() def searchSingle(self, group): if self.isDisabled(): return diff --git a/couchpotato/core/providers/info/_modifier/main.py b/couchpotato/core/providers/info/_modifier/main.py index 598a01d7..cefecb74 100644 --- a/couchpotato/core/providers/info/_modifier/main.py +++ b/couchpotato/core/providers/info/_modifier/main.py @@ -104,7 +104,7 @@ class MovieResultModifier(Plugin): except: log.error('Tried getting more info on searched movies: %s', traceback.format_exc()) - db.close() + pass #db.close() return temp def checkLibrary(self, result): diff --git a/couchpotato/core/providers/metadata/base.py b/couchpotato/core/providers/metadata/base.py index 72d07609..3f336be4 100644 --- a/couchpotato/core/providers/metadata/base.py +++ b/couchpotato/core/providers/metadata/base.py @@ -90,6 +90,7 @@ class MetaDataBase(Plugin): file_type = {} for ft in file_types: + # TODO: change type to "image_"+wanted_file_type if ft.get('identifier') == wanted_file_type: file_type = ft break diff --git a/couchpotato/core/settings/__init__.py b/couchpotato/core/settings/__init__.py index e6565d07..bbbc9381 100644 --- a/couchpotato/core/settings/__init__.py +++ b/couchpotato/core/settings/__init__.py @@ -4,6 +4,7 @@ from couchpotato.api import addApiView from couchpotato.core.event import addEvent, fireEvent from couchpotato.core.helpers.encoding import toUnicode from couchpotato.core.helpers.variable import mergeDicts, tryInt, tryFloat +from couchpotato.core.settings.index import PropertyIndex from couchpotato.core.settings.model import Properties import ConfigParser @@ -42,6 +43,7 @@ class Settings(object): } }"""} }) + addApiView('settings.save', self.saveView, docs = { 'desc': 'Save setting to config file (settings.conf)', 'params': { @@ -51,6 +53,8 @@ class Settings(object): } }) + addEvent('database.setup', self.databaseSetup) + def setFile(self, config_file): self.file = config_file @@ -62,6 +66,17 @@ class Settings(object): self.connectEvents() + def databaseSetup(self): + from couchpotato import get_db + + db = get_db() + + try: + db.add_index(PropertyIndex(db.path, 'property')) + except: + self.log.debug('Index already exists') + db.edit_index(PropertyIndex(db.path, 'property')) + def parser(self): return self.p @@ -206,36 +221,33 @@ class Settings(object): } def getProperty(self, identifier): - from couchpotato import get_session + from couchpotato import get_db - db = get_session() + db = get_db() prop = None try: - propert = db.query(Properties).filter_by(identifier = identifier).first() - prop = propert.value + propert = db.get('property', identifier, with_doc = True) + prop = propert['doc']['value'] except: - pass + self.log.debug('Property doesn\'t exist: %s', traceback.format_exc(0)) - db.close() return prop def setProperty(self, identifier, value = ''): - from couchpotato import get_session + from couchpotato import get_db + + db = get_db() try: - db = get_session() - - p = db.query(Properties).filter_by(identifier = identifier).first() - if not p: - p = Properties() - db.add(p) - - p.identifier = identifier - p.value = toUnicode(value) - - db.commit() + p = db.get('property', identifier, with_doc = True) + p['doc'].update({ + 'identifier': identifier, + 'value': toUnicode(value), + }) + db.update(p['doc']) except: - self.log.error('Failed: %s', traceback.format_exc()) - db.rollback() - finally: - db.close() + db.insert({ + 'type': 'property', + 'identifier': identifier, + 'value': toUnicode(value), + }) diff --git a/couchpotato/core/settings/index.py b/couchpotato/core/settings/index.py new file mode 100644 index 00000000..45d366eb --- /dev/null +++ b/couchpotato/core/settings/index.py @@ -0,0 +1,16 @@ +from CodernityDB.hash_index import UniqueHashIndex, HashIndex +from hashlib import md5 + + +class PropertyIndex(HashIndex): + + def __init__(self, *args, **kwargs): + kwargs['key_format'] = '16s' + super(PropertyIndex, self).__init__(*args, **kwargs) + + def make_key(self, key): + return md5(key).digest() + + def make_key_value(self, data): + if data.get('type') == 'property': + return md5(data['identifier']).digest(), None diff --git a/couchpotato/environment.py b/couchpotato/environment.py index 42f8fded..5eb1272c 100644 --- a/couchpotato/environment.py +++ b/couchpotato/environment.py @@ -31,7 +31,7 @@ class Env(object): _app_dir = "" _data_dir = "" _cache_dir = "" - _db_path = "" + _db = "" _log_path = "" @staticmethod @@ -54,28 +54,6 @@ class Env(object): def set(attr, value): return setattr(Env, '_' + attr, value) - @staticmethod - def getSession(): - existing_session = Env.get('session') - if existing_session: - return existing_session() - - session = scoped_session(sessionmaker(bind = Env.getEngine())) - Env.set('session', session) - - return session() - - @staticmethod - def getEngine(): - existing_engine = Env.get('engine') - if existing_engine: - return existing_engine - - engine = create_engine(Env.get('db_path'), echo = False) - Env.set('engine', engine) - - return engine - @staticmethod def setting(attr, section = 'core', value = None, default = '', type = None): diff --git a/couchpotato/runner.py b/couchpotato/runner.py index bd033983..744d425b 100644 --- a/couchpotato/runner.py +++ b/couchpotato/runner.py @@ -1,3 +1,4 @@ +from CodernityDB.database_thread_safe import ThreadSafeDatabase from argparse import ArgumentParser from cache import FileSystemCache from couchpotato import KeyHandler, LoginHandler, LogoutHandler @@ -82,52 +83,25 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En Env.set('encoding', encoding) # Do db stuff - db_path = toUnicode(os.path.join(data_dir, 'couchpotato.db')) + db_path = toUnicode(os.path.join(data_dir, 'database')) - # Backup before start and cleanup old databases - new_backup = toUnicode(os.path.join(data_dir, 'db_backup', str(int(time.time())))) - if not os.path.isdir(new_backup): os.makedirs(new_backup) + # Check if database exists + print db_path - # Remove older backups, keep backups 3 days or at least 3 - backups = [] - for directory in os.listdir(os.path.dirname(new_backup)): - backup = toUnicode(os.path.join(os.path.dirname(new_backup), directory)) - if os.path.isdir(backup): - backups.append(backup) + db = ThreadSafeDatabase(db_path) + db_exists = db.exists() + if db_exists: + db.open() + else: + db.create() - latest_backup = tryInt(os.path.basename(sorted(backups)[-1])) if len(backups) > 0 else 0 - if latest_backup < time.time() - 3600: - # Create path and copy - src_files = [options.config_file, db_path, db_path + '-shm', db_path + '-wal'] - for src_file in src_files: - if os.path.isfile(src_file): - dst_file = toUnicode(os.path.join(new_backup, os.path.basename(src_file))) - shutil.copyfile(src_file, dst_file) - - # Try and copy stats seperately - try: shutil.copystat(src_file, dst_file) - except: pass - - total_backups = len(backups) - for backup in backups: - if total_backups > 3: - if tryInt(os.path.basename(backup)) < time.time() - 259200: - for the_file in os.listdir(backup): - file_path = os.path.join(backup, the_file) - try: - if os.path.isfile(file_path): - os.remove(file_path) - except: - raise - - os.rmdir(backup) - total_backups -= 1 + # TODO:Backup before start and cleanup old databases # Register environment settings Env.set('app_dir', toUnicode(base_path)) Env.set('data_dir', toUnicode(data_dir)) Env.set('log_path', toUnicode(os.path.join(log_dir, 'CouchPotato.log'))) - Env.set('db_path', toUnicode('sqlite:///' + db_path)) + Env.set('db', db) Env.set('cache_dir', toUnicode(os.path.join(data_dir, 'cache'))) Env.set('cache', FileSystemCache(toUnicode(os.path.join(Env.get('cache_dir'), 'python')))) Env.set('console_log', options.console_log) @@ -183,34 +157,6 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En log.warning('%s %s %s line:%s', (category, message, filename, lineno)) warnings.showwarning = customwarn - # Check if database exists - db = Env.get('db_path') - db_exists = os.path.isfile(toUnicode(db_path)) - - # Load migrations - if False and db_exists: - - from migrate.versioning.api import version_control, db_version, version, upgrade - repo = os.path.join(base_path, 'couchpotato', 'core', 'migration') - - latest_db_version = version(repo) - try: - current_db_version = db_version(db, repo) - except: - version_control(db, repo, version = latest_db_version) - current_db_version = db_version(db, repo) - - if current_db_version < latest_db_version: - if development: - log.error('There is a database migration ready, but you are running development mode, so it won\'t be used. If you see this, you are stupid. Please disable development mode.') - else: - log.info('Doing database upgrade. From %d to %d', (current_db_version, latest_db_version)) - upgrade(db, repo) - - # Configure Database - from couchpotato.core.settings.model import setup - setup() - # Create app from couchpotato import WebHandler web_base = ('/' + Env.setting('url_base').lstrip('/') + '/') if Env.setting('url_base') else '/' @@ -279,6 +225,7 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En # Fill database with needed stuff if not db_exists: fireEvent('app.initialize', in_order = True) + fireEvent('database.setup') # Go go go! from tornado.ioloop import IOLoop @@ -287,7 +234,7 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En # Some logging and fire load event try: log.info('Starting server on port %(port)s', config) except: pass - fireEventAsync('app.load') + fireEventAsync('app.load2') if config['ssl_cert'] and config['ssl_key']: server = HTTPServer(application, no_keep_alive = True, ssl_options = { diff --git a/couchpotato/templates/index.html b/couchpotato/templates/index.html index 52a4491b..af257454 100644 --- a/couchpotato/templates/index.html +++ b/couchpotato/templates/index.html @@ -71,8 +71,6 @@ Status.setup({{ json_encode(fireEvent('status.all', single = True)) }}); - File.Type.setup({{ json_encode(fireEvent('file.types', single = True)) }}); - CategoryList.setup({{ json_encode(fireEvent('category.all', single = True)) }}); App.setup({ @@ -95,4 +93,4 @@ CouchPotato - \ No newline at end of file + diff --git a/libs/CodernityDB/__init__.py b/libs/CodernityDB/__init__.py new file mode 100644 index 00000000..8399a60f --- /dev/null +++ b/libs/CodernityDB/__init__.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright 2011-2013 Codernity (http://codernity.com) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +__version__ = '0.4.2' +__license__ = "Apache 2.0" diff --git a/libs/CodernityDB/database.py b/libs/CodernityDB/database.py new file mode 100644 index 00000000..064836f1 --- /dev/null +++ b/libs/CodernityDB/database.py @@ -0,0 +1,1214 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright 2011-2013 Codernity (http://codernity.com) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import io +from inspect import getsource + +# for custom indexes +from CodernityDB.storage import Storage, IU_Storage +from CodernityDB.hash_index import (IU_UniqueHashIndex, + IU_HashIndex, + HashIndex, + UniqueHashIndex) +# normal imports + +from CodernityDB.index import (ElemNotFound, + DocIdNotFound, + IndexException, + Index, + TryReindexException, + ReindexException, + IndexNotFoundException, + IndexConflict) + +from CodernityDB.misc import NONE + +from CodernityDB.env import cdb_environment + +from random import randrange + +import warnings + + +def header_for_indexes(index_name, index_class, db_custom="", ind_custom="", classes_code=""): + return """# %s +# %s + +# inserted automatically +import os +import marshal + +import struct +import shutil + +from hashlib import md5 + +# custom db code start +# db_custom +%s + +# custom index code start +# ind_custom +%s + +# source of classes in index.classes_code +# classes_code +%s + +# index code start + +""" % (index_name, index_class, db_custom, ind_custom, classes_code) + + +class DatabaseException(Exception): + pass + + +class PreconditionsException(DatabaseException): + pass + + +class RecordDeleted(DatabaseException): + pass + + +class RecordNotFound(DatabaseException): + pass + + +class RevConflict(DatabaseException): + pass + + +class DatabaseConflict(DatabaseException): + pass + + +class DatabasePathException(DatabaseException): + pass + + +class DatabaseIsNotOpened(PreconditionsException): + pass + + +class Database(object): + """ + A default single thread database object. + """ + + custom_header = "" # : use it for imports required by your database + + def __init__(self, path): + self.path = path + self.storage = None + self.indexes = [] + self.id_ind = None + self.indexes_names = {} + self.opened = False + + def create_new_rev(self, old_rev=None): + """ + Creates new revision number based on previous one. + Increments it + random bytes. On overflow starts from 0 again. + """ + if old_rev: + try: + rev_num = int(old_rev[:4], 16) + except: + raise RevConflict() + rev_num += 1 + if rev_num > 65025: + # starting the counter from 0 again + rev_num = 0 + rnd = randrange(65536) + return "%04x%04x" % (rev_num, rnd) + else: + # new rev + rnd = randrange(256 ** 2) + return '0001%04x' % rnd + + def __not_opened(self): + if not self.opened: + raise DatabaseIsNotOpened("Database is not opened") + + def set_indexes(self, indexes=[]): + """ + Set indexes using ``indexes`` param + + :param indexes: indexes to set in db + :type indexes: iterable of :py:class:`CodernityDB.index.Index` objects. + + """ + for ind in indexes: + self.add_index(ind, create=False) + + def _add_single_index(self, p, i, index): + """ + Adds single index to a database. + It will use :py:meth:`inspect.getsource` to get class source. + Then it will build real index file, save it in ``_indexes`` directory. + """ + code = getsource(index.__class__) + if not code.startswith('c'): # fix for indented index codes + import textwrap + code = textwrap.dedent(code) + index._order = i + cls_code = getattr(index, 'classes_code', []) + classes_code = "" + for curr in cls_code: + classes_code += getsource(curr) + '\n\n' + with io.FileIO(os.path.join(p, "%.2d%s" % (i, index.name) + '.py'), 'w') as f: + f.write(header_for_indexes(index.name, + index.__class__.__name__, + getattr(self, 'custom_header', ''), + getattr(index, 'custom_header', ''), + classes_code)) + f.write(code) + return True + + def _read_index_single(self, p, ind, ind_kwargs={}): + """ + It will read single index from index file (ie. generated in :py:meth:`._add_single_index`). + Then it will perform ``exec`` on that code + + If error will occur the index file will be saved with ``_broken`` suffix + + :param p: path + :param ind: index name (will be joined with *p*) + :returns: new index object + """ + with io.FileIO(os.path.join(p, ind), 'r') as f: + name = f.readline()[2:].strip() + _class = f.readline()[2:].strip() + code = f.read() + try: + obj = compile(code, '', f.__name__, repr(args[1:]) + res = f(*args, **kwargs) +# if db.opened: +# db.flush() +# print '<=', f.__name__, repr(args[1:]) + return res + return _inner + + def __new__(cls, classname, bases, attr): + new_attr = {} + for base in bases: + for b_attr in dir(base): + a = getattr(base, b_attr, None) + if isinstance(a, MethodType) and not b_attr.startswith('_'): + if b_attr == 'flush' or b_attr == 'flush_indexes': + pass + else: + # setattr(base, b_attr, SuperLock.wrapper(a)) + new_attr[b_attr] = SuperLock.wrapper(a) + for attr_name, attr_value in attr.iteritems(): + if isinstance(attr_value, FunctionType) and not attr_name.startswith('_'): + attr_value = SuperLock.wrapper(attr_value) + new_attr[attr_name] = attr_value + new_attr['super_lock'] = RLock() + return type.__new__(cls, classname, bases, new_attr) + + +class SuperThreadSafeDatabase(Database): + """ + Thread safe version that always allows single thread to use db. + It adds the same lock for all methods, so only one operation can be + performed in given time. Completely different implementation + than ThreadSafe version (without super word) + """ + + __metaclass__ = SuperLock + + def __init__(self, *args, **kwargs): + super(SuperThreadSafeDatabase, self).__init__(*args, **kwargs) + + def __patch_index_gens(self, name): + ind = self.indexes_names[name] + for c in ('all', 'get_many'): + m = getattr(ind, c) + if getattr(ind, c + "_orig", None): + return + m_fixed = th_safe_gen.wrapper(m, name, c, self.super_lock) + setattr(ind, c, m_fixed) + setattr(ind, c + '_orig', m) + + def open(self, *args, **kwargs): + res = super(SuperThreadSafeDatabase, self).open(*args, **kwargs) + for name in self.indexes_names.iterkeys(): + self.__patch_index_gens(name) + return res + + def create(self, *args, **kwargs): + res = super(SuperThreadSafeDatabase, self).create(*args, **kwargs) + for name in self.indexes_names.iterkeys(): + self.__patch_index_gens(name) + return res + + def add_index(self, *args, **kwargs): + res = super(SuperThreadSafeDatabase, self).add_index(*args, **kwargs) + self.__patch_index_gens(res) + return res + + def edit_index(self, *args, **kwargs): + res = super(SuperThreadSafeDatabase, self).edit_index(*args, **kwargs) + self.__patch_index_gens(res) + return res diff --git a/libs/CodernityDB/database_thread_safe.py b/libs/CodernityDB/database_thread_safe.py new file mode 100644 index 00000000..5349e09b --- /dev/null +++ b/libs/CodernityDB/database_thread_safe.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright 2011-2013 Codernity (http://codernity.com) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from threading import RLock + +from CodernityDB.env import cdb_environment + +cdb_environment['mode'] = "threads" +cdb_environment['rlock_obj'] = RLock + + +from database_safe_shared import SafeDatabase + + +class ThreadSafeDatabase(SafeDatabase): + """ + Thread safe version of CodernityDB that uses several lock objects, + on different methods / different indexes etc. It's completely different + implementation of locking than SuperThreadSafe one. + """ + pass diff --git a/libs/CodernityDB/debug_stuff.py b/libs/CodernityDB/debug_stuff.py new file mode 100644 index 00000000..76cdedf9 --- /dev/null +++ b/libs/CodernityDB/debug_stuff.py @@ -0,0 +1,211 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright 2011-2013 Codernity (http://codernity.com) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from CodernityDB.tree_index import TreeBasedIndex +import struct +import os + +import inspect +from functools import wraps +import json + + +class DebugTreeBasedIndex(TreeBasedIndex): + + def __init__(self, *args, **kwargs): + super(DebugTreeBasedIndex, self).__init__(*args, **kwargs) + + def print_tree(self): + print '-----CURRENT TREE-----' + print self.root_flag + + if self.root_flag == 'l': + print '---ROOT---' + self._print_leaf_data(self.data_start) + return + else: + print '---ROOT---' + self._print_node_data(self.data_start) + nr_of_el, children_flag = self._read_node_nr_of_elements_and_children_flag( + self.data_start) + nodes = [] + for index in range(nr_of_el): + l_pointer, key, r_pointer = self._read_single_node_key( + self.data_start, index) + nodes.append(l_pointer) + nodes.append(r_pointer) + print 'ROOT NODES', nodes + while children_flag == 'n': + self._print_level(nodes, 'n') + new_nodes = [] + for node in nodes: + nr_of_el, children_flag = \ + self._read_node_nr_of_elements_and_children_flag(node) + for index in range(nr_of_el): + l_pointer, key, r_pointer = self._read_single_node_key( + node, index) + new_nodes.append(l_pointer) + new_nodes.append(r_pointer) + nodes = new_nodes + self._print_level(nodes, 'l') + + def _print_level(self, nodes, flag): + print '---NEXT LVL---' + if flag == 'n': + for node in nodes: + self._print_node_data(node) + elif flag == 'l': + for node in nodes: + self._print_leaf_data(node) + + def _print_leaf_data(self, leaf_start_position): + print 'printing data of leaf at', leaf_start_position + nr_of_elements = self._read_leaf_nr_of_elements(leaf_start_position) + self.buckets.seek(leaf_start_position) + data = self.buckets.read(self.leaf_heading_size + + nr_of_elements * self.single_leaf_record_size) + leaf = struct.unpack('<' + self.leaf_heading_format + + nr_of_elements * self.single_leaf_record_format, data) + print leaf + print + + def _print_node_data(self, node_start_position): + print 'printing data of node at', node_start_position + nr_of_elements = self._read_node_nr_of_elements_and_children_flag( + node_start_position)[0] + self.buckets.seek(node_start_position) + data = self.buckets.read(self.node_heading_size + self.pointer_size + + nr_of_elements * (self.key_size + self.pointer_size)) + node = struct.unpack('<' + self.node_heading_format + self.pointer_format + + nr_of_elements * ( + self.key_format + self.pointer_format), + data) + print node + print +# ------------------> + + +def database_step_by_step(db_obj, path=None): + + if not path: + # ugly for multiplatform support.... + p = db_obj.path + p1 = os.path.split(p) + p2 = os.path.split(p1[0]) + p3 = '_'.join([p2[1], 'operation_logger.log']) + path = os.path.join(os.path.split(p2[0])[0], p3) + f_obj = open(path, 'wb') + + __stack = [] # inspect.stack() is not working on pytest etc + + def remove_from_stack(name): + for i in range(len(__stack)): + if __stack[-i] == name: + __stack.pop(-i) + + def __dumper(f): + @wraps(f) + def __inner(*args, **kwargs): + funct_name = f.__name__ + if funct_name == 'count': + name = args[0].__name__ + meth_args = (name,) + args[1:] + elif funct_name in ('reindex_index', 'compact_index'): + name = args[0].name + meth_args = (name,) + args[1:] + else: + meth_args = args + kwargs_copy = kwargs.copy() + res = None + __stack.append(funct_name) + if funct_name == 'insert': + try: + res = f(*args, **kwargs) + except: + packed = json.dumps((funct_name, + meth_args, kwargs_copy, None)) + f_obj.write('%s\n' % packed) + f_obj.flush() + raise + else: + packed = json.dumps((funct_name, + meth_args, kwargs_copy, res)) + f_obj.write('%s\n' % packed) + f_obj.flush() + else: + if funct_name == 'get': + for curr in __stack: + if ('delete' in curr or 'update' in curr) and not curr.startswith('test'): + remove_from_stack(funct_name) + return f(*args, **kwargs) + packed = json.dumps((funct_name, meth_args, kwargs_copy)) + f_obj.write('%s\n' % packed) + f_obj.flush() + res = f(*args, **kwargs) + remove_from_stack(funct_name) + return res + return __inner + + for meth_name, meth_f in inspect.getmembers(db_obj, predicate=inspect.ismethod): + if not meth_name.startswith('_'): + setattr(db_obj, meth_name, __dumper(meth_f)) + + setattr(db_obj, 'operation_logger', f_obj) + + +def database_from_steps(db_obj, path): + # db_obj.insert=lambda data : insert_for_debug(db_obj, data) + with open(path, 'rb') as f_obj: + for current in f_obj: + line = json.loads(current[:-1]) + if line[0] == 'count': + obj = getattr(db_obj, line[1][0]) + line[1] = [obj] + line[1][1:] + name = line[0] + if name == 'insert': + try: + line[1][0].pop('_rev') + except: + pass + elif name in ('delete', 'update'): + el = db_obj.get('id', line[1][0]['_id']) + line[1][0]['_rev'] = el['_rev'] +# print 'FROM STEPS doing', line + meth = getattr(db_obj, line[0], None) + if not meth: + raise Exception("Method = `%s` not found" % line[0]) + + meth(*line[1], **line[2]) + + +# def insert_for_debug(self, data): +# +# _rev = data['_rev'] +# +# if not '_id' in data: +# _id = uuid4().hex +# else: +# _id = data['_id'] +# data['_id'] = _id +# try: +# _id = bytes(_id) +# except: +# raise DatabaseException("`_id` must be valid bytes object") +# self._insert_indexes(_id, _rev, data) +# ret = {'_id': _id, '_rev': _rev} +# data.update(ret) +# return ret diff --git a/libs/CodernityDB/env.py b/libs/CodernityDB/env.py new file mode 100644 index 00000000..69ca8cdd --- /dev/null +++ b/libs/CodernityDB/env.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright 2011-2013 Codernity (http://codernity.com) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +It's CodernityDB environment. +Handles internal informations.' +""" + +cdb_environment = { + 'mode': 'normal' +} diff --git a/libs/CodernityDB/hash_index.py b/libs/CodernityDB/hash_index.py new file mode 100644 index 00000000..cd160fd0 --- /dev/null +++ b/libs/CodernityDB/hash_index.py @@ -0,0 +1,880 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright 2011-2013 Codernity (http://codernity.com) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from CodernityDB.index import (Index, + IndexException, + DocIdNotFound, + ElemNotFound, + TryReindexException, + IndexPreconditionsException) + +import os +import marshal +import io +import struct +import shutil + +from CodernityDB.storage import IU_Storage, DummyStorage + +from CodernityDB.env import cdb_environment + +if cdb_environment.get('rlock_obj'): + from CodernityDB import patch + patch.patch_cache_rr(cdb_environment['rlock_obj']) + +from CodernityDB.rr_cache import cache1lvl + + +from CodernityDB.misc import random_hex_32 + +try: + from CodernityDB import __version__ +except ImportError: + from __init__ import __version__ + + +class IU_HashIndex(Index): + """ + That class is for Internal Use only, if you want to use HashIndex just subclass the :py:class:`HashIndex` instead this one. + + That design is because main index logic should be always in database not in custom user indexes. + """ + + def __init__(self, db_path, name, entry_line_format='<32s{key}IIcI', hash_lim=0xfffff, storage_class=None, key_format='c'): + """ + The index is capable to solve conflicts by `Separate chaining` + :param db_path: database path + :type db_path: string + :param name: index name + :type name: ascii string + :param line_format: line format, `key_format` parameter value will replace `{key}` if present. + :type line_format: string (32s{key}IIcI by default) {doc_id}{hash_key}{start}{size}{status}{next} + :param hash_lim: maximum hash functon results (remember about birthday problem) count from 0 + :type hash_lim: integer + :param storage_class: Storage class by default it will open standard :py:class:`CodernityDB.storage.Storage` (if string has to be accesible by globals()[storage_class]) + :type storage_class: class name which will be instance of CodernityDB.storage.Storage instance or None + :param key_format: a index key format + """ + if key_format and '{key}' in entry_line_format: + entry_line_format = entry_line_format.replace('{key}', key_format) + super(IU_HashIndex, self).__init__(db_path, name) + self.hash_lim = hash_lim + if not storage_class: + storage_class = IU_Storage + if storage_class and not isinstance(storage_class, basestring): + storage_class = storage_class.__name__ + self.storage_class = storage_class + self.storage = None + + self.bucket_line_format = "= self.data_start: + self.buckets.seek(pos_prev) + data = self.buckets.read(self.entry_line_size) + if data: + doc_id, l_key, start, size, status, _next = self.entry_struct.unpack(data) + self.buckets.seek(pos_prev) + self.buckets.write(self.entry_struct.pack(doc_id, + l_key, + start, + size, + status, + pos_next)) + self.flush() + if pos_next: + self.buckets.seek(pos_next) + data = self.buckets.read(self.entry_line_size) + if data: + doc_id, l_key, start, size, status, _next = self.entry_struct.unpack(data) + self.buckets.seek(pos_next) + self.buckets.write(self.entry_struct.pack(doc_id, + l_key, + start, + size, + status, + _next)) + self.flush() + return + + def delete(self, doc_id, key, start=0, size=0): + start_position = self._calculate_position(key) + self.buckets.seek(start_position) + curr_data = self.buckets.read(self.bucket_line_size) + if curr_data: + location = self.bucket_struct.unpack(curr_data)[0] + else: + # case happens when trying to delete element with new index key in data + # after adding new index to database without reindex + raise TryReindexException() + found_at, _doc_id, _key, start, size, status, _next = self._locate_doc_id(doc_id, key, location) + self.buckets.seek(found_at) + self.buckets.write(self.entry_struct.pack(doc_id, + key, + start, + size, + 'd', + _next)) + self.flush() + # self._fix_link(_key, _prev, _next) + self._find_key.delete(key) + self._locate_doc_id.delete(doc_id) + return True + + def compact(self, hash_lim=None): + + if not hash_lim: + hash_lim = self.hash_lim + + compact_ind = self.__class__( + self.db_path, self.name + '_compact', hash_lim=hash_lim) + compact_ind.create_index() + + gen = self.all() + while True: + try: + doc_id, key, start, size, status = gen.next() + except StopIteration: + break + self.storage._f.seek(start) + value = self.storage._f.read(size) + start_ = compact_ind.storage._f.tell() + compact_ind.storage._f.write(value) + compact_ind.insert(doc_id, key, start_, size, status) + + compact_ind.close_index() + original_name = self.name + # os.unlink(os.path.join(self.db_path, self.name + "_buck")) + self.close_index() + shutil.move(os.path.join(compact_ind.db_path, compact_ind. + name + "_buck"), os.path.join(self.db_path, self.name + "_buck")) + shutil.move(os.path.join(compact_ind.db_path, compact_ind. + name + "_stor"), os.path.join(self.db_path, self.name + "_stor")) + # self.name = original_name + self.open_index() # reload... + self.name = original_name + self._save_params(dict(name=original_name)) + self._fix_params() + self._clear_cache() + return True + + def make_key(self, key): + return key + + def make_key_value(self, data): + return '1', data + + def _clear_cache(self): + self._find_key.clear() + self._locate_doc_id.clear() + + def close_index(self): + super(IU_HashIndex, self).close_index() + self._clear_cache() + + +class IU_UniqueHashIndex(IU_HashIndex): + """ + Index for *unique* keys! Designed to be a **id** index. + + That class is for Internal Use only, if you want to use UniqueHashIndex just subclass the :py:class:`UniqueHashIndex` instead this one. + + That design is because main index logic should be always in database not in custom user indexes. + """ + + def __init__(self, db_path, name, entry_line_format="<32s8sIIcI", *args, **kwargs): + if 'key' in kwargs: + raise IndexPreconditionsException( + "UniqueHashIndex doesn't accept key parameter'") + super(IU_UniqueHashIndex, self).__init__(db_path, name, + entry_line_format, *args, **kwargs) + self.create_key = random_hex_32 # : set the function to create random key when no _id given + # self.entry_struct=struct.Struct(entry_line_format) + +# @lfu_cache(100) + def _find_key(self, key): + """ + Find the key position + + :param key: the key to find + """ + start_position = self._calculate_position(key) + self.buckets.seek(start_position) + curr_data = self.buckets.read(self.bucket_line_size) + if curr_data: + location = self.bucket_struct.unpack(curr_data)[0] + found_at, l_key, rev, start, size, status, _next = self._locate_key( + key, location) + return l_key, rev, start, size, status + else: + return None, None, 0, 0, 'u' + + def _find_key_many(self, *args, **kwargs): + raise NotImplementedError() + + def _find_place(self, start, key): + """ + Find a place to where put the key. It will iterate using `next` field in record, until + empty `next` found + + :param start: position to start from + """ + location = start + while True: + self.buckets.seek(location) + data = self.buckets.read(self.entry_line_size) + # todo, maybe partial read there... + l_key, rev, start, size, status, _next = self.entry_struct.unpack( + data) + if l_key == key: + raise IndexException("The '%s' key already exists" % key) + if not _next or status == 'd': + return self.buckets.tell() - self.entry_line_size, l_key, rev, start, size, status, _next + else: + location = _next # go to next record + + # @lfu_cache(100) + def _locate_key(self, key, start): + """ + Locate position of the key, it will iterate using `next` field in record + until required key will be find. + + :param key: the key to locate + :param start: position to start from + """ + location = start + while True: + self.buckets.seek(location) + data = self.buckets.read(self.entry_line_size) + # todo, maybe partial read there... + try: + l_key, rev, start, size, status, _next = self.entry_struct.unpack(data) + except struct.error: + raise ElemNotFound("Location '%s' not found" % key) + if l_key == key: + break + else: + if not _next: + # not found + raise ElemNotFound("Location '%s' not found" % key) + else: + location = _next # go to next record + return self.buckets.tell() - self.entry_line_size, l_key, rev, start, size, status, _next + + def update(self, key, rev, u_start=0, u_size=0, u_status='o'): + start_position = self._calculate_position(key) + self.buckets.seek(start_position) + curr_data = self.buckets.read(self.bucket_line_size) + # test if it's unique or not really unique hash + + if curr_data: + location = self.bucket_struct.unpack(curr_data)[0] + else: + raise ElemNotFound("Location '%s' not found" % key) + found_at, _key, _rev, start, size, status, _next = self._locate_key( + key, location) + if u_start == 0: + u_start = start + if u_size == 0: + u_size = size + self.buckets.seek(found_at) + self.buckets.write(self.entry_struct.pack(key, + rev, + u_start, + u_size, + u_status, + _next)) + self.flush() + self._find_key.delete(key) + return True + + def insert(self, key, rev, start, size, status='o'): + start_position = self._calculate_position(key) + self.buckets.seek(start_position) + curr_data = self.buckets.read(self.bucket_line_size) + + # conflict occurs? + if curr_data: + location = self.bucket_struct.unpack(curr_data)[0] + else: + location = 0 + if location: + # last key with that hash + found_at, _key, _rev, _start, _size, _status, _next = self._find_place( + location, key) + self.buckets.seek(0, 2) + wrote_at = self.buckets.tell() + + # check if position is bigger than all hash entries... + if wrote_at < self.data_start: + self.buckets.seek(self.data_start) + wrote_at = self.buckets.tell() + + self.buckets.write(self.entry_struct.pack(key, + rev, + start, + size, + status, + _next)) + +# self.flush() + self.buckets.seek(found_at) + self.buckets.write(self.entry_struct.pack(_key, + _rev, + _start, + _size, + _status, + wrote_at)) + self.flush() + self._find_key.delete(_key) + # self._locate_key.delete(_key) + return True + # raise NotImplementedError + else: + self.buckets.seek(0, 2) + wrote_at = self.buckets.tell() + + # check if position is bigger than all hash entries... + if wrote_at < self.data_start: + self.buckets.seek(self.data_start) + wrote_at = self.buckets.tell() + + self.buckets.write(self.entry_struct.pack(key, + rev, + start, + size, + status, + 0)) +# self.flush() + self.buckets.seek(start_position) + self.buckets.write(self.bucket_struct.pack(wrote_at)) + self.flush() + self._find_key.delete(key) + return True + + def all(self, limit=-1, offset=0): + self.buckets.seek(self.data_start) + while offset: + curr_data = self.buckets.read(self.entry_line_size) + if not curr_data: + break + try: + doc_id, rev, start, size, status, next = self.entry_struct.unpack(curr_data) + except IndexException: + break + else: + if status != 'd': + offset -= 1 + + while limit: + curr_data = self.buckets.read(self.entry_line_size) + if not curr_data: + break + try: + doc_id, rev, start, size, status, next = self.entry_struct.unpack(curr_data) + except IndexException: + break + else: + if status != 'd': + yield doc_id, rev, start, size, status + limit -= 1 + + def get_many(self, *args, **kwargs): + raise NotImplementedError() + + def delete(self, key, start=0, size=0): + self.update(key, '00000000', start, size, 'd') + + def make_key_value(self, data): + _id = data['_id'] + try: + _id = bytes(data['_id']) + except: + raise IndexPreconditionsException( + "_id must be valid string/bytes object") + if len(_id) != 32: + raise IndexPreconditionsException("Invalid _id lenght") + del data['_id'] + del data['_rev'] + return _id, data + + def destroy(self): + Index.destroy(self) + self._clear_cache() + + def _clear_cache(self): + self._find_key.clear() + + def insert_with_storage(self, _id, _rev, value): + if value: + start, size = self.storage.insert(value) + else: + start = 1 + size = 0 + return self.insert(_id, _rev, start, size) + + def update_with_storage(self, _id, _rev, value): + if value: + start, size = self.storage.insert(value) + else: + start = 1 + size = 0 + return self.update(_id, _rev, start, size) + + +class DummyHashIndex(IU_HashIndex): + def __init__(self, db_path, name, entry_line_format="<32s4sIIcI", *args, **kwargs): + super(DummyHashIndex, self).__init__(db_path, name, + entry_line_format, *args, **kwargs) + self.create_key = random_hex_32 # : set the function to create random key when no _id given + # self.entry_struct=struct.Struct(entry_line_format) + + def update(self, *args, **kwargs): + return True + + def insert(self, *args, **kwargs): + return True + + def all(self, *args, **kwargs): + raise StopIteration + + def get(self, *args, **kwargs): + raise ElemNotFound + + def get_many(self, *args, **kwargs): + raise StopIteration + + def delete(self, *args, **kwargs): + pass + + def make_key_value(self, data): + return '1', {'_': 1} + + def destroy(self): + pass + + def _clear_cache(self): + pass + + def _open_storage(self): + if not self.storage: + self.storage = DummyStorage() + self.storage.open() + + def _create_storage(self): + if not self.storage: + self.storage = DummyStorage() + self.storage.create() + + +class IU_MultiHashIndex(IU_HashIndex): + """ + Class that allows to index more than one key per database record. + + It operates very well on GET/INSERT. It's not optimized for + UPDATE operations (will always readd everything) + """ + + def __init__(self, *args, **kwargs): + super(IU_MultiHashIndex, self).__init__(*args, **kwargs) + + def insert(self, doc_id, key, start, size, status='o'): + if isinstance(key, (list, tuple)): + key = set(key) + elif not isinstance(key, set): + key = set([key]) + ins = super(IU_MultiHashIndex, self).insert + for curr_key in key: + ins(doc_id, curr_key, start, size, status) + return True + + def update(self, doc_id, key, u_start, u_size, u_status='o'): + if isinstance(key, (list, tuple)): + key = set(key) + elif not isinstance(key, set): + key = set([key]) + upd = super(IU_MultiHashIndex, self).update + for curr_key in key: + upd(doc_id, curr_key, u_start, u_size, u_status) + + def delete(self, doc_id, key, start=0, size=0): + if isinstance(key, (list, tuple)): + key = set(key) + elif not isinstance(key, set): + key = set([key]) + delete = super(IU_MultiHashIndex, self).delete + for curr_key in key: + delete(doc_id, curr_key, start, size) + + def get(self, key): + return super(IU_MultiHashIndex, self).get(key) + + def make_key_value(self, data): + raise NotImplementedError() + + +# classes for public use, done in this way because of +# generation static files with indexes (_index directory) + + +class HashIndex(IU_HashIndex): + """ + That class is designed to be used in custom indexes. + """ + pass + + +class UniqueHashIndex(IU_UniqueHashIndex): + """ + That class is designed to be used in custom indexes. It's designed to be **id** index. + """ + pass + + +class MultiHashIndex(IU_MultiHashIndex): + """ + That class is designed to be used in custom indexes. + """ diff --git a/libs/CodernityDB/index.py b/libs/CodernityDB/index.py new file mode 100644 index 00000000..48db2a4a --- /dev/null +++ b/libs/CodernityDB/index.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright 2011-2013 Codernity (http://codernity.com) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +import marshal + +import struct +import shutil + +from CodernityDB.storage import IU_Storage, DummyStorage + +try: + from CodernityDB import __version__ +except ImportError: + from __init__ import __version__ + + +import io + + +class IndexException(Exception): + pass + + +class IndexNotFoundException(IndexException): + pass + + +class ReindexException(IndexException): + pass + + +class TryReindexException(ReindexException): + pass + + +class ElemNotFound(IndexException): + pass + + +class DocIdNotFound(ElemNotFound): + pass + + +class IndexConflict(IndexException): + pass + + +class IndexPreconditionsException(IndexException): + pass + + +class Index(object): + + __version__ = __version__ + + custom_header = "" # : use it for imports required by your index + + def __init__(self, + db_path, + name): + self.name = name + self._start_ind = 500 + self.db_path = db_path + + def open_index(self): + if not os.path.isfile(os.path.join(self.db_path, self.name + '_buck')): + raise IndexException("Doesn't exists") + self.buckets = io.open( + os.path.join(self.db_path, self.name + "_buck"), 'r+b', buffering=0) + self._fix_params() + self._open_storage() + + def _close(self): + self.buckets.close() + self.storage.close() + + def close_index(self): + self.flush() + self.fsync() + self._close() + + def create_index(self): + raise NotImplementedError() + + def _fix_params(self): + self.buckets.seek(0) + props = marshal.loads(self.buckets.read(self._start_ind)) + for k, v in props.iteritems(): + self.__dict__[k] = v + self.buckets.seek(0, 2) + + def _save_params(self, in_params={}): + self.buckets.seek(0) + props = marshal.loads(self.buckets.read(self._start_ind)) + props.update(in_params) + self.buckets.seek(0) + data = marshal.dumps(props) + if len(data) > self._start_ind: + raise IndexException("To big props") + self.buckets.write(data) + self.flush() + self.buckets.seek(0, 2) + self.__dict__.update(props) + + def _open_storage(self, *args, **kwargs): + pass + + def _create_storage(self, *args, **kwargs): + pass + + def _destroy_storage(self, *args, **kwargs): + self.storage.destroy() + + def _find_key(self, key): + raise NotImplementedError() + + def update(self, doc_id, key, start, size): + raise NotImplementedError() + + def insert(self, doc_id, key, start, size): + raise NotImplementedError() + + def get(self, key): + raise NotImplementedError() + + def get_many(self, key, start_from=None, limit=0): + raise NotImplementedError() + + def all(self, start_pos): + raise NotImplementedError() + + def delete(self, key, start, size): + raise NotImplementedError() + + def make_key_value(self, data): + raise NotImplementedError() + + def make_key(self, data): + raise NotImplementedError() + + def compact(self, *args, **kwargs): + raise NotImplementedError() + + def destroy(self, *args, **kwargs): + self._close() + bucket_file = os.path.join(self.db_path, self.name + '_buck') + os.unlink(bucket_file) + self._destroy_storage() + self._find_key.clear() + + def flush(self): + try: + self.buckets.flush() + self.storage.flush() + except: + pass + + def fsync(self): + try: + os.fsync(self.buckets.fileno()) + self.storage.fsync() + except: + pass + + def update_with_storage(self, doc_id, key, value): + if value: + start, size = self.storage.insert(value) + else: + start = 1 + size = 0 + return self.update(doc_id, key, start, size) + + def insert_with_storage(self, doc_id, key, value): + if value: + start, size = self.storage.insert(value) + else: + start = 1 + size = 0 + return self.insert(doc_id, key, start, size) diff --git a/libs/CodernityDB/indexcreator.py b/libs/CodernityDB/indexcreator.py new file mode 100644 index 00000000..1e09a22b --- /dev/null +++ b/libs/CodernityDB/indexcreator.py @@ -0,0 +1,645 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright 2011-2013 Codernity (http://codernity.com) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import re +import tokenize +import token +import uuid + + +class IndexCreatorException(Exception): + def __init__(self, ex, line=None): + self.ex = ex + self.line = line + + def __str__(self): + if self.line: + return repr(self.ex + "(in line: %d)" % self.line) + return repr(self.ex) + + +class IndexCreatorFunctionException(IndexCreatorException): + pass + + +class IndexCreatorValueException(IndexCreatorException): + pass + + +class Parser(object): + def __init__(self): + pass + + def parse(self, data, name=None): + if not name: + self.name = "_" + uuid.uuid4().hex + else: + self.name = name + + self.ind = 0 + self.stage = 0 + self.logic = ['and', 'or', 'in'] + self.logic2 = ['&', '|'] + self.allowed_props = {'TreeBasedIndex': ['type', 'name', 'key_format', 'node_capacity', 'pointer_format', 'meta_format'], + 'HashIndex': ['type', 'name', 'key_format', 'hash_lim', 'entry_line_format'], + 'MultiHashIndex': ['type', 'name', 'key_format', 'hash_lim', 'entry_line_format'], + 'MultiTreeBasedIndex': ['type', 'name', 'key_format', 'node_capacity', 'pointer_format', 'meta_format'] + } + self.funcs = {'md5': (['md5'], ['.digest()']), + 'len': (['len'], []), + 'str': (['str'], []), + 'fix_r': (['self.fix_r'], []), + 'prefix': (['self.prefix'], []), + 'infix': (['self.infix'], []), + 'suffix': (['self.suffix'], []) + } + self.handle_int_imports = {'infix': "from itertools import izip\n"} + + self.funcs_with_body = {'fix_r': + (""" def fix_r(self,s,l): + e = len(s) + if e == l: + return s + elif e > l: + return s[:l] + else: + return s.rjust(l,'_')\n""", False), + 'prefix': + (""" def prefix(self,s,m,l,f): + t = len(s) + if m < 1: + m = 1 + o = set() + if t > l: + s = s[:l] + t = l + while m <= t: + o.add(s.rjust(f,'_')) + s = s[:-1] + t -= 1 + return o\n""", False), + 'suffix': + (""" def suffix(self,s,m,l,f): + t = len(s) + if m < 1: + m = 1 + o = set() + if t > l: + s = s[t-l:] + t = len(s) + while m <= t: + o.add(s.rjust(f,'_')) + s = s[1:] + t -= 1 + return o\n""", False), + 'infix': + (""" def infix(self,s,m,l,f): + t = len(s) + o = set() + for x in xrange(m - 1, l): + t = (s, ) + for y in xrange(0, x): + t += (s[y + 1:],) + o.update(set(''.join(x).rjust(f, '_').lower() for x in izip(*t))) + return o\n""", False)} + self.none = ['None', 'none', 'null'] + self.props_assign = ['=', ':'] + self.all_adj_num_comp = {token.NUMBER: ( + token.NUMBER, token.NAME, '-', '('), + token.NAME: (token.NUMBER, token.NAME, '-', '('), + ')': (token.NUMBER, token.NAME, '-', '(') + } + + self.all_adj_num_op = {token.NUMBER: (token.NUMBER, token.NAME, '('), + token.NAME: (token.NUMBER, token.NAME, '('), + ')': (token.NUMBER, token.NAME, '(') + } + self.allowed_adjacent = { + "<=": self.all_adj_num_comp, + ">=": self.all_adj_num_comp, + ">": self.all_adj_num_comp, + "<": self.all_adj_num_comp, + + "==": {token.NUMBER: (token.NUMBER, token.NAME, '('), + token.NAME: (token.NUMBER, token.NAME, token.STRING, '('), + token.STRING: (token.NAME, token.STRING, '('), + ')': (token.NUMBER, token.NAME, token.STRING, '('), + ']': (token.NUMBER, token.NAME, token.STRING, '(') + }, + + "+": {token.NUMBER: (token.NUMBER, token.NAME, '('), + token.NAME: (token.NUMBER, token.NAME, token.STRING, '('), + token.STRING: (token.NAME, token.STRING, '('), + ')': (token.NUMBER, token.NAME, token.STRING, '('), + ']': (token.NUMBER, token.NAME, token.STRING, '(') + }, + + "-": {token.NUMBER: (token.NUMBER, token.NAME, '('), + token.NAME: (token.NUMBER, token.NAME, '('), + ')': (token.NUMBER, token.NAME, '('), + '<': (token.NUMBER, token.NAME, '('), + '>': (token.NUMBER, token.NAME, '('), + '<=': (token.NUMBER, token.NAME, '('), + '>=': (token.NUMBER, token.NAME, '('), + '==': (token.NUMBER, token.NAME, '('), + ']': (token.NUMBER, token.NAME, '(') + }, + "*": self.all_adj_num_op, + "/": self.all_adj_num_op, + "%": self.all_adj_num_op, + ",": {token.NUMBER: (token.NUMBER, token.NAME, token.STRING, '{', '[', '('), + token.NAME: (token.NUMBER, token.NAME, token.STRING, '(', '{', '['), + token.STRING: (token.NAME, token.STRING, token.NUMBER, '(', '{', '['), + ')': (token.NUMBER, token.NAME, token.STRING, '(', '{', '['), + ']': (token.NUMBER, token.NAME, token.STRING, '(', '{', '['), + '}': (token.NUMBER, token.NAME, token.STRING, '(', '{', '[') + } + } + + def is_num(s): + m = re.search('[^0-9*()+\-\s/]+', s) + return not m + + def is_string(s): + m = re.search('\s*(?P[\'\"]+).*?(?P=a)\s*', s) + return m + data = re.split('make_key_value\:', data) + + if len(data) < 2: + raise IndexCreatorFunctionException( + "Couldn't find a definition of make_key_value function!\n") + + spl1 = re.split('make_key\:', data[0]) + spl2 = re.split('make_key\:', data[1]) + + self.funcs_rev = False + + if len(spl1) > 1: + data = [spl1[0]] + [data[1]] + [spl1[1]] + self.funcs_rev = True + elif len(spl2) > 1: + data = [data[0]] + spl2 + else: + data.append("key") + + if data[1] == re.search('\s*', data[1], re.S | re.M).group(0): + raise IndexCreatorFunctionException("Empty function body ", + len(re.split('\n', data[0])) + (len(re.split('\n', data[2])) if self.funcs_rev else 1) - 1) + if data[2] == re.search('\s*', data[2], re.S | re.M).group(0): + raise IndexCreatorFunctionException("Empty function body ", + len(re.split('\n', data[0])) + (1 if self.funcs_rev else len(re.split('\n', data[1]))) - 1) + if data[0] == re.search('\s*', data[0], re.S | re.M).group(0): + raise IndexCreatorValueException("You didn't set any properity or you set them not at the begining of the code\n") + + data = [re.split( + '\n', data[0]), re.split('\n', data[1]), re.split('\n', data[2])] + self.cnt_lines = (len(data[0]), len(data[1]), len(data[2])) + ind = 0 + self.predata = data + self.data = [[], [], []] + for i, v in enumerate(self.predata[0]): + for k, w in enumerate(self.predata[0][i]): + if self.predata[0][i][k] in self.props_assign: + if not is_num(self.predata[0][i][k + 1:]) and self.predata[0][i].strip()[:4] != 'type' and self.predata[0][i].strip()[:4] != 'name': + s = self.predata[0][i][k + 1:] + self.predata[0][i] = self.predata[0][i][:k + 1] + + m = re.search('\s+', s.strip()) + if not is_string(s) and not m: + s = "'" + s.strip() + "'" + self.predata[0][i] += s + break + + for n, i in enumerate(self.predata): + for k in i: + k = k.strip() + if k: + self.data[ind].append(k) + self.check_enclosures(k, n) + ind += 1 + + return self.parse_ex() + + def readline(self, stage): + def foo(): + if len(self.data[stage]) <= self.ind: + self.ind = 0 + return "" + else: + self.ind += 1 + return self.data[stage][self.ind - 1] + return foo + + def add(self, l, i): + def add_aux(*args): + # print args,self.ind + if len(l[i]) < self.ind: + l[i].append([]) + l[i][self.ind - 1].append(args) + return add_aux + + def parse_ex(self): + self.index_name = "" + self.index_type = "" + self.curLine = -1 + self.con = -1 + self.brackets = -1 + self.curFunc = None + self.colons = 0 + self.line_cons = ([], [], []) + self.pre_tokens = ([], [], []) + self.known_dicts_in_mkv = [] + self.prop_name = True + self.prop_assign = False + self.is_one_arg_enough = False + self.funcs_stack = [] + self.last_line = [-1, -1, -1] + self.props_set = [] + self.custom_header = set() + + self.tokens = [] + self.tokens_head = ['# %s\n' % self.name, 'class %s(' % self.name, '):\n', ' def __init__(self, *args, **kwargs): '] + + for i in xrange(3): + tokenize.tokenize(self.readline(i), self.add(self.pre_tokens, i)) + # tokenize treats some keyword not in the right way, thats why we + # have to change some of them + for nk, k in enumerate(self.pre_tokens[i]): + for na, a in enumerate(k): + if a[0] == token.NAME and a[1] in self.logic: + self.pre_tokens[i][nk][ + na] = (token.OP, a[1], a[2], a[3], a[4]) + + for i in self.pre_tokens[1]: + self.line_cons[1].append(self.check_colons(i, 1)) + self.check_adjacents(i, 1) + if self.check_for_2nd_arg(i) == -1 and not self.is_one_arg_enough: + raise IndexCreatorValueException("No 2nd value to return (did u forget about ',None'?", self.cnt_line_nr(i[0][4], 1)) + self.is_one_arg_enough = False + + for i in self.pre_tokens[2]: + self.line_cons[2].append(self.check_colons(i, 2)) + self.check_adjacents(i, 2) + + for i in self.pre_tokens[0]: + self.handle_prop_line(i) + + self.cur_brackets = 0 + self.tokens += ['\n super(%s, self).__init__(*args, **kwargs)\n def make_key_value(self, data): ' % self.name] + + for i in self.pre_tokens[1]: + for k in i: + self.handle_make_value(*k) + + self.curLine = -1 + self.con = -1 + self.cur_brackets = 0 + self.tokens += ['\n def make_key(self, key):'] + + for i in self.pre_tokens[2]: + for k in i: + self.handle_make_key(*k) + + if self.index_type == "": + raise IndexCreatorValueException("Missing index type definition\n") + if self.index_name == "": + raise IndexCreatorValueException("Missing index name\n") + + self.tokens_head[0] = "# " + self.index_name + "\n" + \ + self.tokens_head[0] + + for i in self.funcs_with_body: + if self.funcs_with_body[i][1]: + self.tokens_head.insert(4, self.funcs_with_body[i][0]) + + if None in self.custom_header: + self.custom_header.remove(None) + if self.custom_header: + s = ' custom_header = """' + for i in self.custom_header: + s += i + s += '"""\n' + self.tokens_head.insert(4, s) + + if self.index_type in self.allowed_props: + for i in self.props_set: + if i not in self.allowed_props[self.index_type]: + raise IndexCreatorValueException("Properity %s is not allowed for index type: %s" % (i, self.index_type)) + + # print "".join(self.tokens_head) + # print "----------" + # print (" ".join(self.tokens)) + return "".join(self.custom_header), "".join(self.tokens_head) + (" ".join(self.tokens)) + + # has to be run BEFORE tokenize + def check_enclosures(self, d, st): + encs = [] + contr = {'(': ')', '{': '}', '[': ']', "'": "'", '"': '"'} + ends = [')', '}', ']', "'", '"'] + for i in d: + if len(encs) > 0 and encs[-1] in ['"', "'"]: + if encs[-1] == i: + del encs[-1] + elif i in contr: + encs += [i] + elif i in ends: + if len(encs) < 1 or contr[encs[-1]] != i: + raise IndexCreatorValueException("Missing opening enclosure for \'%s\'" % i, self.cnt_line_nr(d, st)) + del encs[-1] + + if len(encs) > 0: + raise IndexCreatorValueException("Missing closing enclosure for \'%s\'" % encs[0], self.cnt_line_nr(d, st)) + + def check_adjacents(self, d, st): + def std_check(d, n): + if n == 0: + prev = -1 + else: + prev = d[n - 1][1] if d[n - 1][0] == token.OP else d[n - 1][0] + + cur = d[n][1] if d[n][0] == token.OP else d[n][0] + + # there always is an endmarker at the end, but this is a precaution + if n + 2 > len(d): + nex = -1 + else: + nex = d[n + 1][1] if d[n + 1][0] == token.OP else d[n + 1][0] + + if prev not in self.allowed_adjacent[cur]: + raise IndexCreatorValueException("Wrong left value of the %s" % cur, self.cnt_line_nr(line, st)) + + # there is an assumption that whole data always ends with 0 marker, the idea prolly needs a rewritting to allow more whitespaces + # between tokens, so it will be handled anyway + elif nex not in self.allowed_adjacent[cur][prev]: + raise IndexCreatorValueException("Wrong right value of the %s" % cur, self.cnt_line_nr(line, st)) + + for n, (t, i, _, _, line) in enumerate(d): + if t == token.NAME or t == token.STRING: + if n + 1 < len(d) and d[n + 1][0] in [token.NAME, token.STRING]: + raise IndexCreatorValueException("Did you forget about an operator in between?", self.cnt_line_nr(line, st)) + elif i in self.allowed_adjacent: + std_check(d, n) + + def check_colons(self, d, st): + cnt = 0 + br = 0 + + def check_ret_args_nr(a, s): + c_b_cnt = 0 + s_b_cnt = 0 + n_b_cnt = 0 + comas_cnt = 0 + for _, i, _, _, line in a: + + if c_b_cnt == n_b_cnt == s_b_cnt == 0: + if i == ',': + comas_cnt += 1 + if (s == 1 and comas_cnt > 1) or (s == 2 and comas_cnt > 0): + raise IndexCreatorFunctionException("Too much arguments to return", self.cnt_line_nr(line, st)) + if s == 0 and comas_cnt > 0: + raise IndexCreatorValueException("A coma here doesn't make any sense", self.cnt_line_nr(line, st)) + + elif i == ':': + if s == 0: + raise IndexCreatorValueException("A colon here doesn't make any sense", self.cnt_line_nr(line, st)) + raise IndexCreatorFunctionException("Two colons don't make any sense", self.cnt_line_nr(line, st)) + + if i == '{': + c_b_cnt += 1 + elif i == '}': + c_b_cnt -= 1 + elif i == '(': + n_b_cnt += 1 + elif i == ')': + n_b_cnt -= 1 + elif i == '[': + s_b_cnt += 1 + elif i == ']': + s_b_cnt -= 1 + + def check_if_empty(a): + for i in a: + if i not in [token.NEWLINE, token.INDENT, token.ENDMARKER]: + return False + return True + if st == 0: + check_ret_args_nr(d, st) + return + + for n, i in enumerate(d): + if i[1] == ':': + if br == 0: + if len(d) < n or check_if_empty(d[n + 1:]): + raise IndexCreatorValueException( + "Empty return value", self.cnt_line_nr(i[4], st)) + elif len(d) >= n: + check_ret_args_nr(d[n + 1:], st) + return cnt + else: + cnt += 1 + elif i[1] == '{': + br += 1 + elif i[1] == '}': + br -= 1 + check_ret_args_nr(d, st) + return -1 + + def check_for_2nd_arg(self, d): + c_b_cnt = 0 # curly brackets counter '{}' + s_b_cnt = 0 # square brackets counter '[]' + n_b_cnt = 0 # normal brackets counter '()' + + def check_2nd_arg(d, ind): + d = d[ind[0]:] + for t, i, (n, r), _, line in d: + if i == '{' or i is None: + return 0 + elif t == token.NAME: + self.known_dicts_in_mkv.append((i, (n, r))) + return 0 + elif t == token.STRING or t == token.NUMBER: + raise IndexCreatorValueException("Second return value of make_key_value function has to be a dictionary!", self.cnt_line_nr(line, 1)) + + for ind in enumerate(d): + t, i, _, _, _ = ind[1] + if s_b_cnt == n_b_cnt == c_b_cnt == 0: + if i == ',': + return check_2nd_arg(d, ind) + elif (t == token.NAME and i not in self.funcs) or i == '{': + self.is_one_arg_enough = True + + if i == '{': + c_b_cnt += 1 + self.is_one_arg_enough = True + elif i == '}': + c_b_cnt -= 1 + elif i == '(': + n_b_cnt += 1 + elif i == ')': + n_b_cnt -= 1 + elif i == '[': + s_b_cnt += 1 + elif i == ']': + s_b_cnt -= 1 + return -1 + + def cnt_line_nr(self, l, stage): + nr = -1 + for n, i in enumerate(self.predata[stage]): + # print i,"|||",i.strip(),"|||",l + if l == i.strip(): + nr = n + if nr == -1: + return -1 + + if stage == 0: + return nr + 1 + elif stage == 1: + return nr + self.cnt_lines[0] + (self.cnt_lines[2] - 1 if self.funcs_rev else 0) + elif stage == 2: + return nr + self.cnt_lines[0] + (self.cnt_lines[1] - 1 if not self.funcs_rev else 0) + + return -1 + + def handle_prop_line(self, d): + d_len = len(d) + if d[d_len - 1][0] == token.ENDMARKER: + d_len -= 1 + + if d_len < 3: + raise IndexCreatorValueException("Can't handle properity assingment ", self.cnt_line_nr(d[0][4], 0)) + + if not d[1][1] in self.props_assign: + raise IndexCreatorValueException( + "Did you forget : or =?", self.cnt_line_nr(d[0][4], 0)) + + if d[0][0] == token.NAME or d[0][0] == token.STRING: + if d[0][1] in self.props_set: + raise IndexCreatorValueException("Properity %s is set more than once" % d[0][1], self.cnt_line_nr(d[0][4], 0)) + self.props_set += [d[0][1]] + if d[0][1] == "type" or d[0][1] == "name": + t, tk, _, _, line = d[2] + + if d_len > 3: + raise IndexCreatorValueException( + "Wrong value to assign", self.cnt_line_nr(line, 0)) + + if t == token.STRING: + m = re.search('\s*(?P[\'\"]+)(.*?)(?P=a)\s*', tk) + if m: + tk = m.groups()[1] + elif t != token.NAME: + raise IndexCreatorValueException( + "Wrong value to assign", self.cnt_line_nr(line, 0)) + + if d[0][1] == "type": + if d[2][1] == "TreeBasedIndex": + self.custom_header.add("from CodernityDB.tree_index import TreeBasedIndex\n") + elif d[2][1] == "MultiTreeBasedIndex": + self.custom_header.add("from CodernityDB.tree_index import MultiTreeBasedIndex\n") + elif d[2][1] == "MultiHashIndex": + self.custom_header.add("from CodernityDB.hash_index import MultiHashIndex\n") + self.tokens_head.insert(2, tk) + self.index_type = tk + else: + self.index_name = tk + return + else: + self.tokens += ['\n kwargs["' + d[0][1] + '"]'] + else: + raise IndexCreatorValueException("Can't handle properity assingment ", self.cnt_line_nr(d[0][4], 0)) + + self.tokens += ['='] + + self.check_adjacents(d[2:], 0) + self.check_colons(d[2:], 0) + + for i in d[2:]: + self.tokens += [i[1]] + + def generate_func(self, t, tk, pos_start, pos_end, line, hdata, stage): + if self.last_line[stage] != -1 and pos_start[0] > self.last_line[stage] and line != '': + raise IndexCreatorFunctionException("This line will never be executed!", self.cnt_line_nr(line, stage)) + if t == 0: + return + + if pos_start[1] == 0: + if self.line_cons[stage][pos_start[0] - 1] == -1: + self.tokens += ['\n return'] + self.last_line[stage] = pos_start[0] + else: + self.tokens += ['\n if'] + elif tk == ':' and self.line_cons[stage][pos_start[0] - 1] > -1: + if self.line_cons[stage][pos_start[0] - 1] == 0: + self.tokens += [':\n return'] + return + self.line_cons[stage][pos_start[0] - 1] -= 1 + + if tk in self.logic2: + # print tk + if line[pos_start[1] - 1] != tk and line[pos_start[1] + 1] != tk: + self.tokens += [tk] + if line[pos_start[1] - 1] != tk and line[pos_start[1] + 1] == tk: + if tk == '&': + self.tokens += ['and'] + else: + self.tokens += ['or'] + return + + if self.brackets != 0: + def search_through_known_dicts(a): + for i, (n, r) in self.known_dicts_in_mkv: + if i == tk and r > pos_start[1] and n == pos_start[0] and hdata == 'data': + return True + return False + + if t == token.NAME and len(self.funcs_stack) > 0 and self.funcs_stack[-1][0] == 'md5' and search_through_known_dicts(tk): + raise IndexCreatorValueException("Second value returned by make_key_value for sure isn't a dictionary ", self.cnt_line_nr(line, 1)) + + if tk == ')': + self.cur_brackets -= 1 + if len(self.funcs_stack) > 0 and self.cur_brackets == self.funcs_stack[-1][1]: + self.tokens += [tk] + self.tokens += self.funcs[self.funcs_stack[-1][0]][1] + del self.funcs_stack[-1] + return + if tk == '(': + self.cur_brackets += 1 + + if tk in self.none: + self.tokens += ['None'] + return + + if t == token.NAME and tk not in self.logic and tk != hdata: + if tk not in self.funcs: + self.tokens += [hdata + '["' + tk + '"]'] + else: + self.tokens += self.funcs[tk][0] + if tk in self.funcs_with_body: + self.funcs_with_body[tk] = ( + self.funcs_with_body[tk][0], True) + self.custom_header.add(self.handle_int_imports.get(tk)) + self.funcs_stack += [(tk, self.cur_brackets)] + else: + self.tokens += [tk] + + def handle_make_value(self, t, tk, pos_start, pos_end, line): + self.generate_func(t, tk, pos_start, pos_end, line, 'data', 1) + + def handle_make_key(self, t, tk, pos_start, pos_end, line): + self.generate_func(t, tk, pos_start, pos_end, line, 'key', 2) diff --git a/libs/CodernityDB/lfu_cache.py b/libs/CodernityDB/lfu_cache.py new file mode 100644 index 00000000..e11ffc95 --- /dev/null +++ b/libs/CodernityDB/lfu_cache.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright 2011-2013 Codernity (http://codernity.com) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import functools +from heapq import nsmallest +from operator import itemgetter +from collections import defaultdict + +try: + from collections import Counter +except ImportError: + class Counter(dict): + 'Mapping where default values are zero' + def __missing__(self, key): + return 0 + + +def cache1lvl(maxsize=100): + """ + modified version of http://code.activestate.com/recipes/498245/ + """ + def decorating_function(user_function): + cache = {} + use_count = Counter() + + @functools.wraps(user_function) + def wrapper(key, *args, **kwargs): + try: + result = cache[key] + except KeyError: + if len(cache) == maxsize: + for k, _ in nsmallest(maxsize // 10 or 1, + use_count.iteritems(), + key=itemgetter(1)): + del cache[k], use_count[k] + cache[key] = user_function(key, *args, **kwargs) + result = cache[key] + # result = user_function(obj, key, *args, **kwargs) + finally: + use_count[key] += 1 + return result + + def clear(): + cache.clear() + use_count.clear() + + def delete(key): + try: + del cache[key] + del use_count[key] + except KeyError: + return False + else: + return True + + wrapper.clear = clear + wrapper.cache = cache + wrapper.delete = delete + return wrapper + return decorating_function + + +def twolvl_iterator(dict): + for k, v in dict.iteritems(): + for kk, vv in v.iteritems(): + yield k, kk, vv + + +def cache2lvl(maxsize=100): + """ + modified version of http://code.activestate.com/recipes/498245/ + """ + def decorating_function(user_function): + cache = {} + use_count = defaultdict(Counter) + + @functools.wraps(user_function) + def wrapper(*args, **kwargs): +# return user_function(*args, **kwargs) + try: + result = cache[args[0]][args[1]] + except KeyError: + if wrapper.cache_size == maxsize: + to_delete = maxsize // 10 or 1 + for k1, k2, v in nsmallest(to_delete, + twolvl_iterator(use_count), + key=itemgetter(2)): + del cache[k1][k2], use_count[k1][k2] + if not cache[k1]: + del cache[k1] + del use_count[k1] + wrapper.cache_size -= to_delete + result = user_function(*args, **kwargs) + try: + cache[args[0]][args[1]] = result + except KeyError: + cache[args[0]] = {args[1]: result} + wrapper.cache_size += 1 + finally: + use_count[args[0]][args[1]] += 1 + return result + + def clear(): + cache.clear() + use_count.clear() + + def delete(key, inner_key=None): + if inner_key is not None: + try: + del cache[key][inner_key] + del use_count[key][inner_key] + if not cache[key]: + del cache[key] + del use_count[key] + wrapper.cache_size -= 1 + except KeyError: + return False + else: + return True + else: + try: + wrapper.cache_size -= len(cache[key]) + del cache[key] + del use_count[key] + except KeyError: + return False + else: + return True + + wrapper.clear = clear + wrapper.cache = cache + wrapper.delete = delete + wrapper.cache_size = 0 + return wrapper + return decorating_function diff --git a/libs/CodernityDB/lfu_cache_with_lock.py b/libs/CodernityDB/lfu_cache_with_lock.py new file mode 100644 index 00000000..39f43cc6 --- /dev/null +++ b/libs/CodernityDB/lfu_cache_with_lock.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright 2011-2013 Codernity (http://codernity.com) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import functools +from heapq import nsmallest +from operator import itemgetter +from collections import defaultdict + + +try: + from collections import Counter +except ImportError: + class Counter(dict): + 'Mapping where default values are zero' + def __missing__(self, key): + return 0 + + +def twolvl_iterator(dict): + for k, v in dict.iteritems(): + for kk, vv in v.iteritems(): + yield k, kk, vv + + +def create_cache1lvl(lock_obj): + def cache1lvl(maxsize=100): + """ + modified version of http://code.activestate.com/recipes/498245/ + """ + def decorating_function(user_function): + cache = {} + use_count = Counter() + lock = lock_obj() + + @functools.wraps(user_function) + def wrapper(key, *args, **kwargs): + try: + result = cache[key] + except KeyError: + with lock: + if len(cache) == maxsize: + for k, _ in nsmallest(maxsize // 10 or 1, + use_count.iteritems(), + key=itemgetter(1)): + del cache[k], use_count[k] + cache[key] = user_function(key, *args, **kwargs) + result = cache[key] + use_count[key] += 1 + else: + with lock: + use_count[key] += 1 + return result + + def clear(): + cache.clear() + use_count.clear() + + def delete(key): + try: + del cache[key] + del use_count[key] + return True + except KeyError: + return False + + wrapper.clear = clear + wrapper.cache = cache + wrapper.delete = delete + return wrapper + return decorating_function + return cache1lvl + + +def create_cache2lvl(lock_obj): + def cache2lvl(maxsize=100): + """ + modified version of http://code.activestate.com/recipes/498245/ + """ + def decorating_function(user_function): + cache = {} + use_count = defaultdict(Counter) + lock = lock_obj() + + @functools.wraps(user_function) + def wrapper(*args, **kwargs): + try: + result = cache[args[0]][args[1]] + except KeyError: + with lock: + if wrapper.cache_size == maxsize: + to_delete = maxsize / 10 or 1 + for k1, k2, v in nsmallest(to_delete, + twolvl_iterator( + use_count), + key=itemgetter(2)): + del cache[k1][k2], use_count[k1][k2] + if not cache[k1]: + del cache[k1] + del use_count[k1] + wrapper.cache_size -= to_delete + result = user_function(*args, **kwargs) + try: + cache[args[0]][args[1]] = result + except KeyError: + cache[args[0]] = {args[1]: result} + use_count[args[0]][args[1]] += 1 + wrapper.cache_size += 1 + else: + use_count[args[0]][args[1]] += 1 + return result + + def clear(): + cache.clear() + use_count.clear() + + def delete(key, *args): + if args: + try: + del cache[key][args[0]] + del use_count[key][args[0]] + if not cache[key]: + del cache[key] + del use_count[key] + wrapper.cache_size -= 1 + return True + except KeyError: + return False + else: + try: + wrapper.cache_size -= len(cache[key]) + del cache[key] + del use_count[key] + return True + except KeyError: + return False + + wrapper.clear = clear + wrapper.cache = cache + wrapper.delete = delete + wrapper.cache_size = 0 + return wrapper + return decorating_function + return cache2lvl diff --git a/libs/CodernityDB/migrate.py b/libs/CodernityDB/migrate.py new file mode 100644 index 00000000..4d0b4005 --- /dev/null +++ b/libs/CodernityDB/migrate.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright 2011-2013 Codernity (http://codernity.com) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from CodernityDB.database import Database +import shutil +import os + + +def migrate(source, destination): + """ + Very basic for now + """ + dbs = Database(source) + dbt = Database(destination) + dbs.open() + dbt.create() + dbt.close() + for curr in os.listdir(os.path.join(dbs.path, '_indexes')): + if curr != '00id.py': + shutil.copyfile(os.path.join(dbs.path, '_indexes', curr), + os.path.join(dbt.path, '_indexes', curr)) + dbt.open() + for c in dbs.all('id'): + del c['_rev'] + dbt.insert(c) + return True + + +if __name__ == '__main__': + import sys + migrate(sys.argv[1], sys.argv[2]) diff --git a/libs/CodernityDB/misc.py b/libs/CodernityDB/misc.py new file mode 100644 index 00000000..54c94812 --- /dev/null +++ b/libs/CodernityDB/misc.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright 2011-2013 Codernity (http://codernity.com) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from random import getrandbits, randrange +import uuid + + +class NONE: + """ + It's inteded to be None but different, + for internal use only! + """ + pass + + +def random_hex_32(): + return uuid.UUID(int=getrandbits(128), version=4).hex + + +def random_hex_4(*args, **kwargs): + return '%04x' % randrange(256 ** 2) diff --git a/libs/CodernityDB/patch.py b/libs/CodernityDB/patch.py new file mode 100644 index 00000000..4c074f43 --- /dev/null +++ b/libs/CodernityDB/patch.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright 2011-2013 Codernity (http://codernity.com) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from CodernityDB.misc import NONE + + +def __patch(obj, name, new): + n = NONE() + orig = getattr(obj, name, n) + if orig is not n: + if orig == new: + raise Exception("Shouldn't happen, new and orig are the same") + setattr(obj, name, new) + return + + +def patch_cache_lfu(lock_obj): + """ + Patnches cache mechanizm to be thread safe (gevent ones also) + + .. note:: + + It's internal CodernityDB mechanizm, it will be called when needed + + """ + import lfu_cache + import lfu_cache_with_lock + lfu_lock1lvl = lfu_cache_with_lock.create_cache1lvl(lock_obj) + lfu_lock2lvl = lfu_cache_with_lock.create_cache2lvl(lock_obj) + __patch(lfu_cache, 'cache1lvl', lfu_lock1lvl) + __patch(lfu_cache, 'cache2lvl', lfu_lock2lvl) + + +def patch_cache_rr(lock_obj): + """ + Patches cache mechanizm to be thread safe (gevent ones also) + + .. note:: + + It's internal CodernityDB mechanizm, it will be called when needed + + """ + import rr_cache + import rr_cache_with_lock + rr_lock1lvl = rr_cache_with_lock.create_cache1lvl(lock_obj) + rr_lock2lvl = rr_cache_with_lock.create_cache2lvl(lock_obj) + __patch(rr_cache, 'cache1lvl', rr_lock1lvl) + __patch(rr_cache, 'cache2lvl', rr_lock2lvl) + + +def patch_flush_fsync(db_obj): + """ + Will always execute index.fsync after index.flush. + + .. note:: + + It's for advanced users, use when you understand difference between `flush` and `fsync`, and when you definitely need that. + + It's important to call it **AFTER** database has all indexes etc (after db.create or db.open) + + Example usage:: + + ... + db = Database('/tmp/patch_demo') + db.create() + patch_flush_fsync(db) + ... + + """ + + def always_fsync(ind_obj): + def _inner(): + ind_obj.orig_flush() + ind_obj.fsync() + return _inner + + for index in db_obj.indexes: + setattr(index, 'orig_flush', index.flush) + setattr(index, 'flush', always_fsync(index)) + + setattr(db_obj, 'orig_flush', db_obj.flush) + setattr(db_obj, 'flush', always_fsync(db_obj)) + + return diff --git a/libs/CodernityDB/rr_cache.py b/libs/CodernityDB/rr_cache.py new file mode 100644 index 00000000..5801b7cc --- /dev/null +++ b/libs/CodernityDB/rr_cache.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright 2011-2013 Codernity (http://codernity.com) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import functools +from random import choice + + +def cache1lvl(maxsize=100): + def decorating_function(user_function): + cache1lvl = {} + + @functools.wraps(user_function) + def wrapper(key, *args, **kwargs): + try: + result = cache1lvl[key] + except KeyError: + if len(cache1lvl) == maxsize: + for i in xrange(maxsize // 10 or 1): + del cache1lvl[choice(cache1lvl.keys())] + cache1lvl[key] = user_function(key, *args, **kwargs) + result = cache1lvl[key] +# result = user_function(obj, key, *args, **kwargs) + return result + + def clear(): + cache1lvl.clear() + + def delete(key): + try: + del cache1lvl[key] + return True + except KeyError: + return False + + wrapper.clear = clear + wrapper.cache = cache1lvl + wrapper.delete = delete + return wrapper + return decorating_function + + +def cache2lvl(maxsize=100): + def decorating_function(user_function): + cache = {} + + @functools.wraps(user_function) + def wrapper(*args, **kwargs): +# return user_function(*args, **kwargs) + try: + result = cache[args[0]][args[1]] + except KeyError: +# print wrapper.cache_size + if wrapper.cache_size == maxsize: + to_delete = maxsize // 10 or 1 + for i in xrange(to_delete): + key1 = choice(cache.keys()) + key2 = choice(cache[key1].keys()) + del cache[key1][key2] + if not cache[key1]: + del cache[key1] + wrapper.cache_size -= to_delete +# print wrapper.cache_size + result = user_function(*args, **kwargs) + try: + cache[args[0]][args[1]] = result + except KeyError: + cache[args[0]] = {args[1]: result} + wrapper.cache_size += 1 + return result + + def clear(): + cache.clear() + wrapper.cache_size = 0 + + def delete(key, inner_key=None): + if inner_key: + try: + del cache[key][inner_key] + if not cache[key]: + del cache[key] + wrapper.cache_size -= 1 + return True + except KeyError: + return False + else: + try: + wrapper.cache_size -= len(cache[key]) + del cache[key] + return True + except KeyError: + return False + + wrapper.clear = clear + wrapper.cache = cache + wrapper.delete = delete + wrapper.cache_size = 0 + return wrapper + return decorating_function diff --git a/libs/CodernityDB/rr_cache_with_lock.py b/libs/CodernityDB/rr_cache_with_lock.py new file mode 100644 index 00000000..66298c59 --- /dev/null +++ b/libs/CodernityDB/rr_cache_with_lock.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright 2011-2013 Codernity (http://codernity.com) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import functools +from random import choice + + +def create_cache1lvl(lock_obj): + def cache1lvl(maxsize=100): + def decorating_function(user_function): + cache = {} + lock = lock_obj() + + @functools.wraps(user_function) + def wrapper(key, *args, **kwargs): + try: + result = cache[key] + except KeyError: + with lock: + if len(cache) == maxsize: + for i in xrange(maxsize // 10 or 1): + del cache[choice(cache.keys())] + cache[key] = user_function(key, *args, **kwargs) + result = cache[key] + return result + + def clear(): + cache.clear() + + def delete(key): + try: + del cache[key] + return True + except KeyError: + return False + + wrapper.clear = clear + wrapper.cache = cache + wrapper.delete = delete + return wrapper + return decorating_function + return cache1lvl + + +def create_cache2lvl(lock_obj): + def cache2lvl(maxsize=100): + def decorating_function(user_function): + cache = {} + lock = lock_obj() + + @functools.wraps(user_function) + def wrapper(*args, **kwargs): + try: + result = cache[args[0]][args[1]] + except KeyError: + with lock: + if wrapper.cache_size == maxsize: + to_delete = maxsize // 10 or 1 + for i in xrange(to_delete): + key1 = choice(cache.keys()) + key2 = choice(cache[key1].keys()) + del cache[key1][key2] + if not cache[key1]: + del cache[key1] + wrapper.cache_size -= to_delete + result = user_function(*args, **kwargs) + try: + cache[args[0]][args[1]] = result + except KeyError: + cache[args[0]] = {args[1]: result} + wrapper.cache_size += 1 + return result + + def clear(): + cache.clear() + wrapper.cache_size = 0 + + def delete(key, *args): + if args: + try: + del cache[key][args[0]] + if not cache[key]: + del cache[key] + wrapper.cache_size -= 1 + return True + except KeyError: + return False + else: + try: + wrapper.cache_size -= len(cache[key]) + del cache[key] + return True + except KeyError: + return False + + wrapper.clear = clear + wrapper.cache = cache + wrapper.delete = delete + wrapper.cache_size = 0 + return wrapper + return decorating_function + return cache2lvl diff --git a/libs/CodernityDB/sharded_hash.py b/libs/CodernityDB/sharded_hash.py new file mode 100644 index 00000000..08a8c2f0 --- /dev/null +++ b/libs/CodernityDB/sharded_hash.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright 2011-2013 Codernity (http://codernity.com) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from CodernityDB.hash_index import UniqueHashIndex, HashIndex +from CodernityDB.sharded_index import ShardedIndex +from CodernityDB.index import IndexPreconditionsException + +from random import getrandbits +import uuid + + +class IU_ShardedUniqueHashIndex(ShardedIndex): + + custom_header = """import uuid +from random import getrandbits +from CodernityDB.sharded_index import ShardedIndex +""" + + def __init__(self, db_path, name, *args, **kwargs): + if kwargs.get('sh_nums', 0) > 255: + raise IndexPreconditionsException("Too many shards") + kwargs['ind_class'] = UniqueHashIndex + super(IU_ShardedUniqueHashIndex, self).__init__(db_path, + name, *args, **kwargs) + self.patchers.append(self.wrap_insert_id_index) + + @staticmethod + def wrap_insert_id_index(db_obj, clean=False): + def _insert_id_index(_rev, data): + """ + Performs insert on **id** index. + """ + _id, value = db_obj.id_ind.make_key_value(data) # may be improved + trg_shard = _id[:2] + storage = db_obj.id_ind.shards_r[trg_shard].storage + start, size = storage.insert(value) + db_obj.id_ind.insert(_id, _rev, start, size) + return _id + if not clean: + if hasattr(db_obj, '_insert_id_index_orig'): + raise IndexPreconditionsException( + "Already patched, something went wrong") + setattr(db_obj, "_insert_id_index_orig", db_obj._insert_id_index) + setattr(db_obj, "_insert_id_index", _insert_id_index) + else: + setattr(db_obj, "_insert_id_index", db_obj._insert_id_index_orig) + delattr(db_obj, "_insert_id_index_orig") + + def create_key(self): + h = uuid.UUID(int=getrandbits(128), version=4).hex + trg = self.last_used + 1 + if trg >= self.sh_nums: + trg = 0 + self.last_used = trg + h = '%02x%30s' % (trg, h[2:]) + return h + + def delete(self, key, *args, **kwargs): + trg_shard = key[:2] + op = self.shards_r[trg_shard] + return op.delete(key, *args, **kwargs) + + def update(self, key, *args, **kwargs): + trg_shard = key[:2] + self.last_used = int(trg_shard, 16) + op = self.shards_r[trg_shard] + return op.update(key, *args, **kwargs) + + def insert(self, key, *args, **kwargs): + trg_shard = key[:2] # in most cases it's in create_key BUT not always + self.last_used = int(key[:2], 16) + op = self.shards_r[trg_shard] + return op.insert(key, *args, **kwargs) + + def get(self, key, *args, **kwargs): + trg_shard = key[:2] + self.last_used = int(trg_shard, 16) + op = self.shards_r[trg_shard] + return op.get(key, *args, **kwargs) + + +class ShardedUniqueHashIndex(IU_ShardedUniqueHashIndex): + + # allow unique hash to be used directly + custom_header = 'from CodernityDB.sharded_hash import IU_ShardedUniqueHashIndex' + + pass + + +class IU_ShardedHashIndex(ShardedIndex): + + custom_header = """from CodernityDB.sharded_index import ShardedIndex""" + + def __init__(self, db_path, name, *args, **kwargs): + kwargs['ind_class'] = HashIndex + super(IU_ShardedHashIndex, self).__init__(db_path, name, * + args, **kwargs) + + def calculate_shard(self, key): + """ + Must be implemented. It has to return shard to be used by key + + :param key: key + :returns: target shard + :rtype: int + """ + raise NotImplementedError() + + def delete(self, doc_id, key, *args, **kwargs): + trg_shard = self.calculate_shard(key) + op = self.shards_r[trg_shard] + return op.delete(doc_id, key, *args, **kwargs) + + def insert(self, doc_id, key, *args, **kwargs): + trg_shard = self.calculate_shard(key) + op = self.shards_r[trg_shard] + return op.insert(doc_id, key, *args, **kwargs) + + def update(self, doc_id, key, *args, **kwargs): + trg_shard = self.calculate_shard(key) + op = self.shards_r[trg_shard] + return op.insert(doc_id, key, *args, **kwargs) + + def get(self, key, *args, **kwargs): + trg_shard = self.calculate_shard(key) + op = self.shards_r[trg_shard] + return op.get(key, *args, **kwargs) + + +class ShardedHashIndex(IU_ShardedHashIndex): + pass diff --git a/libs/CodernityDB/sharded_index.py b/libs/CodernityDB/sharded_index.py new file mode 100644 index 00000000..2bdf9d75 --- /dev/null +++ b/libs/CodernityDB/sharded_index.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright 2011-2013 Codernity (http://codernity.com) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from CodernityDB.index import Index +# from CodernityDB.env import cdb_environment +# import warnings + + +class ShardedIndex(Index): + + def __init__(self, db_path, name, *args, **kwargs): + """ + There are 3 additional parameters. You have to hardcode them in your custom class. **NEVER** use directly + + :param int sh_nums: how many shards should be + :param class ind_class: Index class to use (HashIndex or your custom one) + :param bool use_make_keys: if True, `make_key`, and `make_key_value` will be overriden with those from first shard + + The rest parameters are passed straight to `ind_class` shards. + + """ + super(ShardedIndex, self).__init__(db_path, name) + try: + self.sh_nums = kwargs.pop('sh_nums') + except KeyError: + self.sh_nums = 5 + try: + ind_class = kwargs.pop('ind_class') + except KeyError: + raise Exception("ind_class must be given") + else: + # if not isinstance(ind_class, basestring): + # ind_class = ind_class.__name__ + self.ind_class = ind_class + if 'use_make_keys' in kwargs: + self.use_make_keys = kwargs.pop('use_make_keys') + else: + self.use_make_keys = False + self._set_shard_datas(*args, **kwargs) + self.patchers = [] # database object patchers + + def _set_shard_datas(self, *args, **kwargs): + self.shards = {} + self.shards_r = {} +# ind_class = globals()[self.ind_class] + ind_class = self.ind_class + i = 0 + for sh_name in [self.name + str(x) for x in xrange(self.sh_nums)]: + # dict is better than list in that case + self.shards[i] = ind_class(self.db_path, sh_name, *args, **kwargs) + self.shards_r['%02x' % i] = self.shards[i] + self.shards_r[i] = self.shards[i] + i += 1 + + if not self.use_make_keys: + self.make_key = self.shards[0].make_key + self.make_key_value = self.shards[0].make_key_value + + self.last_used = 0 + + @property + def storage(self): + st = self.shards[self.last_used].storage + return st + + def __getattr__(self, name): + return getattr(self.shards[self.last_used], name) + + def open_index(self): + for curr in self.shards.itervalues(): + curr.open_index() + + def create_index(self): + for curr in self.shards.itervalues(): + curr.create_index() + + def destroy(self): + for curr in self.shards.itervalues(): + curr.destroy() + + def compact(self): + for curr in self.shards.itervalues(): + curr.compact() + + def reindex(self): + for curr in self.shards.itervalues(): + curr.reindex() + + def all(self, *args, **kwargs): + for curr in self.shards.itervalues(): + for now in curr.all(*args, **kwargs): + yield now + + def get_many(self, *args, **kwargs): + for curr in self.shards.itervalues(): + for now in curr.get_many(*args, **kwargs): + yield now diff --git a/libs/CodernityDB/storage.py b/libs/CodernityDB/storage.py new file mode 100644 index 00000000..30be1f3b --- /dev/null +++ b/libs/CodernityDB/storage.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright 2011-2013 Codernity (http://codernity.com) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import struct +import shutil +import marshal +import io + + +try: + from CodernityDB import __version__ +except ImportError: + from __init__ import __version__ + + +class StorageException(Exception): + pass + + +class DummyStorage(object): + """ + Storage mostly used to fake real storage + """ + + def create(self, *args, **kwargs): + pass + + def open(self, *args, **kwargs): + pass + + def close(self, *args, **kwargs): + pass + + def data_from(self, *args, **kwargs): + pass + + def data_to(self, *args, **kwargs): + pass + + def save(self, *args, **kwargs): + return 0, 0 + + def insert(self, *args, **kwargs): + return self.save(*args, **kwargs) + + def update(self, *args, **kwargs): + return 0, 0 + + def get(self, *args, **kwargs): + return None + + # def compact(self, *args, **kwargs): + # pass + + def fsync(self, *args, **kwargs): + pass + + def flush(self, *args, **kwargs): + pass + + +class IU_Storage(object): + + __version__ = __version__ + + def __init__(self, db_path, name='main'): + self.db_path = db_path + self.name = name + self._header_size = 100 + + def create(self): + if os.path.exists(os.path.join(self.db_path, self.name + "_stor")): + raise IOError("Storage already exists!") + with io.open(os.path.join(self.db_path, self.name + "_stor"), 'wb') as f: + f.write(struct.pack("10s90s", self.__version__, '|||||')) + f.close() + self._f = io.open(os.path.join( + self.db_path, self.name + "_stor"), 'r+b', buffering=0) + self.flush() + self._f.seek(0, 2) + + def open(self): + if not os.path.exists(os.path.join(self.db_path, self.name + "_stor")): + raise IOError("Storage doesn't exists!") + self._f = io.open(os.path.join( + self.db_path, self.name + "_stor"), 'r+b', buffering=0) + self.flush() + self._f.seek(0, 2) + + def destroy(self): + os.unlink(os.path.join(self.db_path, self.name + '_stor')) + + def close(self): + self._f.close() + # self.flush() + # self.fsync() + + def data_from(self, data): + return marshal.loads(data) + + def data_to(self, data): + return marshal.dumps(data) + + def save(self, data): + s_data = self.data_to(data) + self._f.seek(0, 2) + start = self._f.tell() + size = len(s_data) + self._f.write(s_data) + self.flush() + return start, size + + def insert(self, data): + return self.save(data) + + def update(self, data): + return self.save(data) + + def get(self, start, size, status='c'): + if status == 'd': + return None + else: + self._f.seek(start) + return self.data_from(self._f.read(size)) + + def flush(self): + self._f.flush() + + def fsync(self): + os.fsync(self._f.fileno()) + + +# classes for public use, done in this way because of +# generation static files with indexes (_index directory) + + +class Storage(IU_Storage): + pass diff --git a/libs/CodernityDB/tree_index.py b/libs/CodernityDB/tree_index.py new file mode 100644 index 00000000..b79805db --- /dev/null +++ b/libs/CodernityDB/tree_index.py @@ -0,0 +1,2048 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright 2011-2013 Codernity (http://codernity.com) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from index import Index, IndexException, DocIdNotFound, ElemNotFound +import struct +import marshal +import os +import io +import shutil +from storage import IU_Storage +# from ipdb import set_trace + +from CodernityDB.env import cdb_environment +from CodernityDB.index import TryReindexException + +if cdb_environment.get('rlock_obj'): + from CodernityDB import patch + patch.patch_cache_rr(cdb_environment['rlock_obj']) + +from CodernityDB.rr_cache import cache1lvl, cache2lvl + +tree_buffer_size = io.DEFAULT_BUFFER_SIZE + +cdb_environment['tree_buffer_size'] = tree_buffer_size + + +MODE_FIRST = 0 +MODE_LAST = 1 + +MOVE_BUFFER_PREV = 0 +MOVE_BUFFER_NEXT = 1 + + +class NodeCapacityException(IndexException): + pass + + +class IU_TreeBasedIndex(Index): + + custom_header = 'from CodernityDB.tree_index import TreeBasedIndex' + + def __init__(self, db_path, name, key_format='32s', pointer_format='I', + meta_format='32sIIc', node_capacity=10, storage_class=None): + if node_capacity < 3: + raise NodeCapacityException + super(IU_TreeBasedIndex, self).__init__(db_path, name) + self.data_start = self._start_ind + 1 + self.node_capacity = node_capacity + self.flag_format = 'c' + self.elements_counter_format = 'h' + self.pointer_format = pointer_format + self.key_format = key_format + self.meta_format = meta_format + self._count_props() + if not storage_class: + storage_class = IU_Storage + if storage_class and not isinstance(storage_class, basestring): + storage_class = storage_class.__name__ + self.storage_class = storage_class + self.storage = None + cache = cache1lvl(100) + twolvl_cache = cache2lvl(150) + self._find_key = cache(self._find_key) + self._match_doc_id = cache(self._match_doc_id) +# self._read_single_leaf_record = +# twolvl_cache(self._read_single_leaf_record) + self._find_key_in_leaf = twolvl_cache(self._find_key_in_leaf) + self._read_single_node_key = twolvl_cache(self._read_single_node_key) + self._find_first_key_occurence_in_node = twolvl_cache( + self._find_first_key_occurence_in_node) + self._find_last_key_occurence_in_node = twolvl_cache( + self._find_last_key_occurence_in_node) + self._read_leaf_nr_of_elements = cache(self._read_leaf_nr_of_elements) + self._read_leaf_neighbours = cache(self._read_leaf_neighbours) + self._read_leaf_nr_of_elements_and_neighbours = cache( + self._read_leaf_nr_of_elements_and_neighbours) + self._read_node_nr_of_elements_and_children_flag = cache( + self._read_node_nr_of_elements_and_children_flag) + + def _count_props(self): + """ + Counts dynamic properties for tree, such as all complex formats + """ + self.single_leaf_record_format = self.key_format + self.meta_format + self.single_node_record_format = self.pointer_format + \ + self.key_format + self.pointer_format + self.node_format = self.elements_counter_format + self.flag_format\ + + self.pointer_format + (self.key_format + + self.pointer_format) * self.node_capacity + self.leaf_format = self.elements_counter_format + self.pointer_format * 2\ + + (self.single_leaf_record_format) * self.node_capacity + self.leaf_heading_format = self.elements_counter_format + \ + self.pointer_format * 2 + self.node_heading_format = self.elements_counter_format + \ + self.flag_format + self.key_size = struct.calcsize('<' + self.key_format) + self.meta_size = struct.calcsize('<' + self.meta_format) + self.single_leaf_record_size = struct.calcsize('<' + self. + single_leaf_record_format) + self.single_node_record_size = struct.calcsize('<' + self. + single_node_record_format) + self.node_size = struct.calcsize('<' + self.node_format) + self.leaf_size = struct.calcsize('<' + self.leaf_format) + self.flag_size = struct.calcsize('<' + self.flag_format) + self.elements_counter_size = struct.calcsize('<' + self. + elements_counter_format) + self.pointer_size = struct.calcsize('<' + self.pointer_format) + self.leaf_heading_size = struct.calcsize( + '<' + self.leaf_heading_format) + self.node_heading_size = struct.calcsize( + '<' + self.node_heading_format) + + def create_index(self): + if os.path.isfile(os.path.join(self.db_path, self.name + '_buck')): + raise IndexException('Already exists') + with io.open(os.path.join(self.db_path, self.name + "_buck"), 'w+b') as f: + props = dict(name=self.name, + flag_format=self.flag_format, + pointer_format=self.pointer_format, + elements_counter_format=self.elements_counter_format, + node_capacity=self.node_capacity, + key_format=self.key_format, + meta_format=self.meta_format, + version=self.__version__, + storage_class=self.storage_class) + f.write(marshal.dumps(props)) + self.buckets = io.open(os.path.join(self.db_path, self.name + + "_buck"), 'r+b', buffering=0) + self._create_storage() + self.buckets.seek(self._start_ind) + self.buckets.write(struct.pack(' candidate_start: + move_buffer = MOVE_BUFFER_PREV + elif buffer_end < candidate_start + self.single_leaf_record_size: + move_buffer = MOVE_BUFFER_NEXT + else: + move_buffer = None + return self._calculate_key_position(leaf_start, (imin + imax) / 2, 'l'), (imin + imax) / 2, move_buffer + + def _choose_next_candidate_index_in_node(self, node_start, candidate_start, buffer_start, buffer_end, imin, imax): + if buffer_start > candidate_start: + move_buffer = MOVE_BUFFER_PREV + elif buffer_end < candidate_start + self.single_node_record_size: + (self.pointer_size + self.key_size) - 1 + move_buffer = MOVE_BUFFER_NEXT + else: + move_buffer = None + return self._calculate_key_position(node_start, (imin + imax) / 2, 'n'), (imin + imax) / 2, move_buffer + + def _find_key_in_leaf(self, leaf_start, key, nr_of_elements): + if nr_of_elements == 1: + return self._find_key_in_leaf_with_one_element(key, leaf_start)[-5:] + else: + return self._find_key_in_leaf_using_binary_search(key, leaf_start, nr_of_elements)[-5:] + + def _find_key_in_leaf_for_update(self, key, doc_id, leaf_start, nr_of_elements): + if nr_of_elements == 1: + return self._find_key_in_leaf_with_one_element(key, leaf_start, doc_id=doc_id) + else: + return self._find_key_in_leaf_using_binary_search(key, leaf_start, nr_of_elements, mode=MODE_FIRST, doc_id=doc_id) + + def _find_index_of_first_key_equal_or_smaller_key(self, key, leaf_start, nr_of_elements): + if nr_of_elements == 1: + return self._find_key_in_leaf_with_one_element(key, leaf_start, mode=MODE_FIRST, return_closest=True)[:2] + else: + return self._find_key_in_leaf_using_binary_search(key, leaf_start, nr_of_elements, mode=MODE_FIRST, return_closest=True)[:2] + + def _find_index_of_last_key_equal_or_smaller_key(self, key, leaf_start, nr_of_elements): + if nr_of_elements == 1: + return self._find_key_in_leaf_with_one_element(key, leaf_start, mode=MODE_LAST, return_closest=True)[:2] + else: + return self._find_key_in_leaf_using_binary_search(key, leaf_start, nr_of_elements, mode=MODE_LAST, return_closest=True)[:2] + + def _find_index_of_first_key_equal(self, key, leaf_start, nr_of_elements): + if nr_of_elements == 1: + return self._find_key_in_leaf_with_one_element(key, leaf_start, mode=MODE_FIRST)[:2] + else: + return self._find_key_in_leaf_using_binary_search(key, leaf_start, nr_of_elements, mode=MODE_FIRST)[:2] + + def _find_key_in_leaf_with_one_element(self, key, leaf_start, doc_id=None, mode=None, return_closest=False): + curr_key, curr_doc_id, curr_start, curr_size,\ + curr_status = self._read_single_leaf_record(leaf_start, 0) + if key != curr_key: + if return_closest and curr_status != 'd': + return leaf_start, 0 + else: + raise ElemNotFound + else: + if curr_status == 'd': + raise ElemNotFound + elif doc_id is not None and doc_id != curr_doc_id: +# should't happen, crashes earlier on id index + raise DocIdNotFound + else: + return leaf_start, 0, curr_doc_id, curr_key, curr_start, curr_size, curr_status + + def _find_key_in_leaf_using_binary_search(self, key, leaf_start, nr_of_elements, doc_id=None, mode=None, return_closest=False): + """ + Binary search implementation used in all get functions + """ + imin, imax = 0, nr_of_elements - 1 + buffer_start, buffer_end = self._set_buffer_limits() + candidate_start, candidate_index, move_buffer = self._choose_next_candidate_index_in_leaf(leaf_start, + self._calculate_key_position(leaf_start, + (imin + imax) / 2, + 'l'), + buffer_start, + buffer_end, + imin, imax) + while imax != imin and imax > imin: + curr_key, curr_doc_id, curr_start, curr_size, curr_status = self._read_single_leaf_record(leaf_start, + candidate_index) + candidate_start = self._calculate_key_position( + leaf_start, candidate_index, 'l') + if key < curr_key: + if move_buffer == MOVE_BUFFER_PREV: + buffer_start, buffer_end = self._prev_buffer( + buffer_start, buffer_end) + else: # if next chosen element is in current buffer, abort moving to other + move_buffer is None + imax = candidate_index - 1 + candidate_start, candidate_index, move_buffer = self._choose_next_candidate_index_in_leaf(leaf_start, + candidate_start, + buffer_start, + buffer_end, + imin, imax) + elif key == curr_key: + if mode == MODE_LAST: + if move_buffer == MOVE_BUFFER_NEXT: + buffer_start, buffer_end = self._next_buffer( + buffer_start, buffer_end) + else: + move_buffer is None + imin = candidate_index + 1 + candidate_start, candidate_index, move_buffer = self._choose_next_candidate_index_in_leaf(leaf_start, + candidate_start, + buffer_start, + buffer_end, + imin, imax) + else: + if curr_status == 'o': + break + else: + if move_buffer == MOVE_BUFFER_PREV: + buffer_start, buffer_end = self._prev_buffer( + buffer_start, buffer_end) + else: + move_buffer is None + imax = candidate_index + candidate_start, candidate_index, move_buffer = self._choose_next_candidate_index_in_leaf(leaf_start, + candidate_start, + buffer_start, + buffer_end, + imin, imax) + else: + if move_buffer == MOVE_BUFFER_NEXT: + buffer_start, buffer_end = self._next_buffer( + buffer_start, buffer_end) + else: + move_buffer is None + imin = candidate_index + 1 + candidate_start, candidate_index, move_buffer = self._choose_next_candidate_index_in_leaf(leaf_start, + candidate_start, + buffer_start, + buffer_end, + imin, imax) + + if imax > imin: + chosen_key_position = candidate_index + else: + chosen_key_position = imax + curr_key, curr_doc_id, curr_start, curr_size, curr_status = self._read_single_leaf_record(leaf_start, + chosen_key_position) + if key != curr_key: + if return_closest: # useful for find all bigger/smaller methods + return leaf_start, chosen_key_position + else: + raise ElemNotFound + if doc_id and doc_id == curr_doc_id and curr_status == 'o': + return leaf_start, chosen_key_position, curr_doc_id, curr_key, curr_start, curr_size, curr_status + else: + if mode == MODE_FIRST and imin < chosen_key_position: # check if there isn't any element with equal key before chosen one + matching_record_index = self._leaf_linear_key_search(key, + self._calculate_key_position(leaf_start, + imin, + 'l'), + imin, + chosen_key_position) + else: + matching_record_index = chosen_key_position + curr_key, curr_doc_id, curr_start, curr_size, curr_status = self._read_single_leaf_record(leaf_start, + matching_record_index) + if curr_status == 'd' and not return_closest: + leaf_start, nr_of_elements, matching_record_index = self._find_existing(key, + matching_record_index, + leaf_start, + nr_of_elements) + curr_key, curr_doc_id, curr_start, curr_size, curr_status = self._read_single_leaf_record(leaf_start, + matching_record_index) + if doc_id is not None and doc_id != curr_doc_id: + leaf_start, nr_of_elements, matching_record_index = self._match_doc_id(doc_id, + key, + matching_record_index, + leaf_start, + nr_of_elements) + curr_key, curr_doc_id, curr_start, curr_size, curr_status = self._read_single_leaf_record(leaf_start, + matching_record_index) + return leaf_start, matching_record_index, curr_doc_id, curr_key, curr_start, curr_size, curr_status + + def _find_place_in_leaf(self, key, leaf_start, nr_of_elements): + if nr_of_elements == 1: + return self._find_place_in_leaf_with_one_element(key, leaf_start) + else: + return self._find_place_in_leaf_using_binary_search(key, leaf_start, nr_of_elements) + + def _find_place_in_leaf_with_one_element(self, key, leaf_start): + curr_key, curr_doc_id, curr_start, curr_size,\ + curr_status = self._read_single_leaf_record(leaf_start, 0) + if curr_status == 'd': + return leaf_start, 0, 0, False, True # leaf start, index of new key position, nr of rec to rewrite, full_leaf flag, on_deleted flag + else: + if key < curr_key: + return leaf_start, 0, 1, False, False + else: + return leaf_start, 1, 0, False, False + + def _find_place_in_leaf_using_binary_search(self, key, leaf_start, nr_of_elements): + """ + Binary search implementation used in insert function + """ + imin, imax = 0, nr_of_elements - 1 + buffer_start, buffer_end = self._set_buffer_limits() + candidate_start, candidate_index, move_buffer = self._choose_next_candidate_index_in_leaf(leaf_start, + self._calculate_key_position(leaf_start, + (imin + imax) / 2, + 'l'), + buffer_start, + buffer_end, + imin, imax) + while imax != imin and imax > imin: + curr_key, curr_doc_id, curr_start, curr_size, curr_status = self._read_single_leaf_record(leaf_start, + candidate_index) + candidate_start = self._calculate_key_position( + leaf_start, candidate_index, 'l') + if key < curr_key: + if move_buffer == MOVE_BUFFER_PREV: + buffer_start, buffer_end = self._prev_buffer( + buffer_start, buffer_end) + else: # if next chosen element is in current buffer, abort moving to other + move_buffer is None + imax = candidate_index - 1 + candidate_start, candidate_index, move_buffer = self._choose_next_candidate_index_in_leaf(leaf_start, + candidate_start, + buffer_start, + buffer_end, + imin, imax) + else: + if move_buffer == MOVE_BUFFER_NEXT: + buffer_start, buffer_end = self._next_buffer( + buffer_start, buffer_end) + else: + move_buffer is None + imin = candidate_index + 1 + candidate_start, candidate_index, move_buffer = self._choose_next_candidate_index_in_leaf(leaf_start, + candidate_start, + buffer_start, + buffer_end, + imin, imax) + if imax < imin and imin < nr_of_elements: + chosen_key_position = imin + else: + chosen_key_position = imax + curr_key, curr_doc_id, curr_start, curr_size, curr_status = self._read_single_leaf_record(leaf_start, + chosen_key_position) + if curr_status == 'd': + return leaf_start, chosen_key_position, 0, False, True + elif key < curr_key: + if chosen_key_position > 0: + curr_key, curr_doc_id, curr_start, curr_size, curr_status = self._read_single_leaf_record(leaf_start, + chosen_key_position - 1) + if curr_start == 'd': + return leaf_start, chosen_key_position - 1, 0, False, True + else: + return leaf_start, chosen_key_position, nr_of_elements - chosen_key_position, (nr_of_elements == self.node_capacity), False + else: + return leaf_start, chosen_key_position, nr_of_elements - chosen_key_position, (nr_of_elements == self.node_capacity), False + else: + if chosen_key_position < nr_of_elements - 1: + curr_key, curr_doc_id, curr_start, curr_size, curr_status = self._read_single_leaf_record(leaf_start, + chosen_key_position + 1) + if curr_start == 'd': + return leaf_start, chosen_key_position + 1, 0, False, True + else: + return leaf_start, chosen_key_position + 1, nr_of_elements - chosen_key_position - 1, (nr_of_elements == self.node_capacity), False + else: + return leaf_start, chosen_key_position + 1, nr_of_elements - chosen_key_position - 1, (nr_of_elements == self.node_capacity), False + + def _set_buffer_limits(self): + pos = self.buckets.tell() + buffer_start = pos - (pos % tree_buffer_size) + return buffer_start, (buffer_start + tree_buffer_size) + + def _find_first_key_occurence_in_node(self, node_start, key, nr_of_elements): + if nr_of_elements == 1: + return self._find_key_in_node_with_one_element(key, node_start, mode=MODE_FIRST) + else: + return self._find_key_in_node_using_binary_search(key, node_start, nr_of_elements, mode=MODE_FIRST) + + def _find_last_key_occurence_in_node(self, node_start, key, nr_of_elements): + if nr_of_elements == 1: + return self._find_key_in_node_with_one_element(key, node_start, mode=MODE_LAST) + else: + return self._find_key_in_node_using_binary_search(key, node_start, nr_of_elements, mode=MODE_LAST) + + def _find_key_in_node_with_one_element(self, key, node_start, mode=None): + l_pointer, curr_key, r_pointer = self._read_single_node_key( + node_start, 0) + if key < curr_key: + return 0, l_pointer + elif key > curr_key: + return 0, r_pointer + else: + if mode == MODE_FIRST: + return 0, l_pointer + elif mode == MODE_LAST: + return 0, r_pointer + else: + raise Exception('Invalid mode declared: set first/last') + + def _find_key_in_node_using_binary_search(self, key, node_start, nr_of_elements, mode=None): + imin, imax = 0, nr_of_elements - 1 + buffer_start, buffer_end = self._set_buffer_limits() + candidate_start, candidate_index, move_buffer = self._choose_next_candidate_index_in_node(node_start, + self._calculate_key_position(node_start, + (imin + imax) / 2, + 'n'), + buffer_start, + buffer_end, + imin, imax) + while imax != imin and imax > imin: + l_pointer, curr_key, r_pointer = self._read_single_node_key( + node_start, candidate_index) + candidate_start = self._calculate_key_position( + node_start, candidate_index, 'n') + if key < curr_key: + if move_buffer == MOVE_BUFFER_PREV: + buffer_start, buffer_end = self._prev_buffer( + buffer_start, buffer_end) + else: # if next chosen element is in current buffer, abort moving to other + move_buffer is None + imax = candidate_index - 1 + candidate_start, candidate_index, move_buffer = self._choose_next_candidate_index_in_node(node_start, + candidate_start, + buffer_start, + buffer_end, + imin, imax) + elif key == curr_key: + if mode == MODE_LAST: + if move_buffer == MOVE_BUFFER_NEXT: + buffer_start, buffer_end = self._next_buffer( + buffer_start, buffer_end) + else: + move_buffer is None + imin = candidate_index + 1 + candidate_start, candidate_index, move_buffer = self._choose_next_candidate_index_in_node(node_start, + candidate_start, + buffer_start, + buffer_end, + imin, imax) + else: + break + else: + if move_buffer == MOVE_BUFFER_NEXT: + buffer_start, buffer_end = self._next_buffer( + buffer_start, buffer_end) + else: + move_buffer is None + imin = candidate_index + 1 + candidate_start, candidate_index, move_buffer = self._choose_next_candidate_index_in_node(node_start, + candidate_start, + buffer_start, + buffer_end, + imin, imax) + + if imax > imin: + chosen_key_position = candidate_index + elif imax < imin and imin < nr_of_elements: + chosen_key_position = imin + else: + chosen_key_position = imax + l_pointer, curr_key, r_pointer = self._read_single_node_key( + node_start, chosen_key_position) + if mode == MODE_FIRST and imin < chosen_key_position: # check if there is no elements with equal key before chosen one + matching_record_index = self._node_linear_key_search(key, + self._calculate_key_position(node_start, + imin, + 'n'), + imin, + chosen_key_position) + else: + matching_record_index = chosen_key_position + l_pointer, curr_key, r_pointer = self._read_single_node_key( + node_start, matching_record_index) + if key < curr_key: + return matching_record_index, l_pointer + elif key > curr_key: + return matching_record_index, r_pointer + else: + if mode == MODE_FIRST: + return matching_record_index, l_pointer + elif mode == MODE_LAST: + return matching_record_index, r_pointer + else: + raise Exception('Invalid mode declared: first/last') + + def _update_leaf_ready_data(self, leaf_start, start_index, new_nr_of_elements, records_to_rewrite): + self.buckets.seek(leaf_start) + self.buckets.write(struct.pack(' new_leaf_size - 1: + key_moved_to_parent_node = leaf_data[(old_leaf_size - 1) * 5] + elif nr_of_records_to_rewrite == new_leaf_size - 1: + key_moved_to_parent_node = new_data[0] + else: + key_moved_to_parent_node = leaf_data[old_leaf_size * 5] + data_to_write = self._prepare_new_root_data(key_moved_to_parent_node, + left_leaf_start_position, + right_leaf_start_position, + 'l') + if nr_of_records_to_rewrite > half_size: + # key goes to first half + # prepare left leaf data + left_leaf_data = struct.pack('<' + self.leaf_heading_format + self.single_leaf_record_format + * (self.node_capacity - nr_of_records_to_rewrite), + old_leaf_size, + 0, + right_leaf_start_position, + *leaf_data[:-nr_of_records_to_rewrite * 5]) + left_leaf_data += struct.pack( + '<' + self.single_leaf_record_format * ( + nr_of_records_to_rewrite - new_leaf_size + 1), + new_data[0], + new_data[1], + new_data[2], + new_data[3], + new_data[4], + *leaf_data[-nr_of_records_to_rewrite * 5:(old_leaf_size - 1) * 5]) + # prepare right leaf_data + right_leaf_data = struct.pack('<' + self.elements_counter_format + 2 * self.pointer_format + + self.single_leaf_record_format * + new_leaf_size, + new_leaf_size, + left_leaf_start_position, + 0, + *leaf_data[-new_leaf_size * 5:]) + else: + # key goes to second half + if nr_of_records_to_rewrite: + records_before = leaf_data[old_leaf_size * + 5:-nr_of_records_to_rewrite * 5] + records_after = leaf_data[-nr_of_records_to_rewrite * 5:] + else: + records_before = leaf_data[old_leaf_size * 5:] + records_after = [] + + left_leaf_data = struct.pack( + '<' + self.leaf_heading_format + + self.single_leaf_record_format * old_leaf_size, + old_leaf_size, + 0, + right_leaf_start_position, + *leaf_data[:old_leaf_size * 5]) + # prepare right leaf_data + right_leaf_data = struct.pack('<' + self.elements_counter_format + 2 * self.pointer_format + + self.single_leaf_record_format * (new_leaf_size - + nr_of_records_to_rewrite - 1), + new_leaf_size, + left_leaf_start_position, + 0, + *records_before) + right_leaf_data += struct.pack( + '<' + self.single_leaf_record_format * ( + nr_of_records_to_rewrite + 1), + new_data[0], + new_data[1], + new_data[2], + new_data[3], + new_data[4], + *records_after) + left_leaf_data += (self.node_capacity - + old_leaf_size) * self.single_leaf_record_size * '\x00' + right_leaf_data += blanks + data_to_write += left_leaf_data + data_to_write += right_leaf_data + self.buckets.seek(self._start_ind) + self.buckets.write(struct.pack(' half_size: # insert key into first half of leaf + self.buckets.seek(self._calculate_key_position(leaf_start, + self.node_capacity - nr_of_records_to_rewrite, + 'l')) + # read all records with key>new_key + data = self.buckets.read( + nr_of_records_to_rewrite * self.single_leaf_record_size) + records_to_rewrite = struct.unpack( + '<' + nr_of_records_to_rewrite * self.single_leaf_record_format, data) + # remove deleted records, if succeded abort spliting + if self._update_if_has_deleted(leaf_start, + records_to_rewrite, + self.node_capacity - + nr_of_records_to_rewrite, + [new_key, new_doc_id, new_start, new_size, new_status]): + return None + key_moved_to_parent_node = records_to_rewrite[ + -new_leaf_size * 5] + # write new leaf at end of file + self.buckets.seek(0, 2) # end of file + new_leaf_start = self.buckets.tell() + # prepare new leaf_data + new_leaf = struct.pack('<' + self.elements_counter_format + 2 * self.pointer_format + + self.single_leaf_record_format * + new_leaf_size, + new_leaf_size, + leaf_start, + next_l, + *records_to_rewrite[-new_leaf_size * 5:]) + new_leaf += blanks + # write new leaf + self.buckets.write(new_leaf) + # update old leaf heading + self._update_leaf_size_and_pointers(leaf_start, + old_leaf_size, + prev_l, + new_leaf_start) + # seek position of new key in first half + self.buckets.seek(self._calculate_key_position(leaf_start, + self.node_capacity - nr_of_records_to_rewrite, + 'l')) + # write new key and keys after + self.buckets.write( + struct.pack( + '<' + self.single_leaf_record_format * + (nr_of_records_to_rewrite - new_leaf_size + 1), + new_key, + new_doc_id, + new_start, + new_size, + 'o', + *records_to_rewrite[:-new_leaf_size * 5])) + + if next_l: # when next_l is 0 there is no next leaf to update, avoids writing data at 0 position of file + self._update_leaf_prev_pointer( + next_l, new_leaf_start) + +# self._read_single_leaf_record.delete(leaf_start) + self._find_key_in_leaf.delete(leaf_start) + + return new_leaf_start, key_moved_to_parent_node + else: # key goes into second half of leaf ' + # seek half of the leaf + self.buckets.seek(self._calculate_key_position( + leaf_start, old_leaf_size, 'l')) + data = self.buckets.read( + self.single_leaf_record_size * (new_leaf_size - 1)) + records_to_rewrite = struct.unpack('<' + (new_leaf_size - 1) * + self.single_leaf_record_format, data) + # remove deleted records, if succeded abort spliting + if self._update_if_has_deleted(leaf_start, + records_to_rewrite, + old_leaf_size, + [new_key, new_doc_id, new_start, new_size, new_status]): + return None + key_moved_to_parent_node = records_to_rewrite[ + -(new_leaf_size - 1) * 5] + if key_moved_to_parent_node > new_key: + key_moved_to_parent_node = new_key + self.buckets.seek(0, 2) # end of file + new_leaf_start = self.buckets.tell() + # prepare new leaf data + index_of_records_split = nr_of_records_to_rewrite * 5 + if index_of_records_split: + records_before = records_to_rewrite[ + :-index_of_records_split] + records_after = records_to_rewrite[ + -index_of_records_split:] + else: + records_before = records_to_rewrite + records_after = [] + new_leaf = struct.pack('<' + self.elements_counter_format + 2 * self.pointer_format + + self.single_leaf_record_format * (new_leaf_size - + nr_of_records_to_rewrite - 1), + new_leaf_size, + leaf_start, + next_l, + *records_before) + new_leaf += struct.pack( + '<' + self.single_leaf_record_format * + (nr_of_records_to_rewrite + 1), + new_key, + new_doc_id, + new_start, + new_size, + 'o', + *records_after) + new_leaf += blanks + self.buckets.write(new_leaf) + self._update_leaf_size_and_pointers(leaf_start, + old_leaf_size, + prev_l, + new_leaf_start) + if next_l: # pren next_l is 0 there is no next leaf to update, avoids writing data at 0 position of file + self._update_leaf_prev_pointer( + next_l, new_leaf_start) + +# self._read_single_leaf_record.delete(leaf_start) + self._find_key_in_leaf.delete(leaf_start) + + return new_leaf_start, key_moved_to_parent_node + + def _update_if_has_deleted(self, leaf_start, records_to_rewrite, start_position, new_record_data): + """ + Checks if there are any deleted elements in data to rewrite and prevent from writing then back. + """ + curr_index = 0 + nr_of_elements = self.node_capacity + records_to_rewrite = list(records_to_rewrite) + for status in records_to_rewrite[4::5]: # remove deleted from list + if status != 'o': + del records_to_rewrite[curr_index * 5:curr_index * 5 + 5] + nr_of_elements -= 1 + else: + curr_index += 1 + # if were deleted dont have to split, just update leaf + if nr_of_elements < self.node_capacity: + data_split_index = 0 + for key in records_to_rewrite[0::5]: + if key > new_record_data[0]: + break + else: + data_split_index += 1 + records_to_rewrite = records_to_rewrite[:data_split_index * 5]\ + + new_record_data\ + + records_to_rewrite[data_split_index * 5:] + self._update_leaf_ready_data(leaf_start, + start_position, + nr_of_elements + 1, + records_to_rewrite), + return True + else: # did not found any deleted records in leaf + return False + + def _prepare_new_root_data(self, root_key, left_pointer, right_pointer, children_flag='n'): + new_root = struct.pack( + '<' + self.node_heading_format + self.single_node_record_format, + 1, + children_flag, + left_pointer, + root_key, + right_pointer) + new_root += (self.key_size + self.pointer_size) * (self. + node_capacity - 1) * '\x00' + return new_root + + def _create_new_root_from_node(self, node_start, children_flag, nr_of_keys_to_rewrite, new_node_size, old_node_size, new_key, new_pointer): + # reading second half of node + self.buckets.seek(self.data_start + self.node_heading_size) + # read all keys with key>new_key + data = self.buckets.read(self.pointer_size + self. + node_capacity * (self.key_size + self.pointer_size)) + old_node_data = struct.unpack('<' + self.pointer_format + self.node_capacity * + (self.key_format + self.pointer_format), data) + self.buckets.seek(0, 2) # end of file + new_node_start = self.buckets.tell() + if nr_of_keys_to_rewrite == new_node_size: + key_moved_to_root = new_key + # prepare new nodes data + left_node = struct.pack('<' + self.node_heading_format + self.pointer_format + + old_node_size * (self. + key_format + self.pointer_format), + old_node_size, + children_flag, + *old_node_data[:old_node_size * 2 + 1]) + + right_node = struct.pack('<' + self.node_heading_format + self.pointer_format + + new_node_size * (self. + key_format + self.pointer_format), + new_node_size, + children_flag, + new_pointer, + *old_node_data[old_node_size * 2 + 1:]) + elif nr_of_keys_to_rewrite > new_node_size: + key_moved_to_root = old_node_data[old_node_size * 2 - 1] + # prepare new nodes data + if nr_of_keys_to_rewrite == self.node_capacity: + keys_before = old_node_data[:1] + keys_after = old_node_data[1:old_node_size * 2 - 1] + else: + keys_before = old_node_data[:-nr_of_keys_to_rewrite * 2] + keys_after = old_node_data[-( + nr_of_keys_to_rewrite) * 2:old_node_size * 2 - 1] + left_node = struct.pack('<' + self.node_heading_format + self.pointer_format + + (self.node_capacity - nr_of_keys_to_rewrite) * (self. + key_format + self.pointer_format), + old_node_size, + children_flag, + *keys_before) + left_node += struct.pack( + '<' + (self.key_format + self.pointer_format) * + (nr_of_keys_to_rewrite - new_node_size), + new_key, + new_pointer, + *keys_after) + + right_node = struct.pack('<' + self.node_heading_format + self.pointer_format + + new_node_size * (self. + key_format + self.pointer_format), + new_node_size, + children_flag, + *old_node_data[old_node_size * 2:]) + else: +# 'inserting key into second half of node and creating new root' + key_moved_to_root = old_node_data[old_node_size * 2 + 1] + # prepare new nodes data + left_node = struct.pack('<' + self.node_heading_format + self.pointer_format + + old_node_size * (self. + key_format + self.pointer_format), + old_node_size, + children_flag, + *old_node_data[:old_node_size * 2 + 1]) + if nr_of_keys_to_rewrite: + keys_before = old_node_data[(old_node_size + + 1) * 2:-nr_of_keys_to_rewrite * 2] + keys_after = old_node_data[-nr_of_keys_to_rewrite * 2:] + else: + keys_before = old_node_data[(old_node_size + 1) * 2:] + keys_after = [] + right_node = struct.pack('<' + self.node_heading_format + self.pointer_format + + (new_node_size - nr_of_keys_to_rewrite - 1) * (self. + key_format + self.pointer_format), + new_node_size, + children_flag, + *keys_before) + right_node += struct.pack( + '<' + (nr_of_keys_to_rewrite + 1) * + (self.key_format + self.pointer_format), + new_key, + new_pointer, + *keys_after) + new_root = self._prepare_new_root_data(key_moved_to_root, + new_node_start, + new_node_start + self.node_size) + left_node += (self.node_capacity - old_node_size) * \ + (self.key_size + self.pointer_size) * '\x00' + # adding blanks after new node + right_node += (self.node_capacity - new_node_size) * \ + (self.key_size + self.pointer_size) * '\x00' + self.buckets.seek(0, 2) + self.buckets.write(left_node + right_node) + self.buckets.seek(self.data_start) + self.buckets.write(new_root) + + self._read_single_node_key.delete(node_start) + self._read_node_nr_of_elements_and_children_flag.delete(node_start) + return None + + def _split_node(self, node_start, nr_of_keys_to_rewrite, new_key, new_pointer, children_flag, create_new_root=False): + """ + Splits full node in two separate ones, first half of records stays on old position, + second half is written as new leaf at the end of file. + """ + half_size = self.node_capacity / 2 + if self.node_capacity % 2 == 0: + old_node_size = new_node_size = half_size + else: + old_node_size = half_size + new_node_size = half_size + 1 + if create_new_root: + self._create_new_root_from_node(node_start, children_flag, nr_of_keys_to_rewrite, new_node_size, old_node_size, new_key, new_pointer) + else: + blanks = (self.node_capacity - new_node_size) * ( + self.key_size + self.pointer_size) * '\x00' + if nr_of_keys_to_rewrite == new_node_size: # insert key into first half of node + # reading second half of node + self.buckets.seek(self._calculate_key_position(node_start, + old_node_size, + 'n') + self.pointer_size) + # read all keys with key>new_key + data = self.buckets.read(nr_of_keys_to_rewrite * + (self.key_size + self.pointer_size)) + old_node_data = struct.unpack('<' + nr_of_keys_to_rewrite * + (self.key_format + self.pointer_format), data) + # write new node at end of file + self.buckets.seek(0, 2) + new_node_start = self.buckets.tell() + # prepare new node_data + new_node = struct.pack('<' + self.node_heading_format + self.pointer_format + + (self.key_format + + self.pointer_format) * new_node_size, + new_node_size, + children_flag, + new_pointer, + *old_node_data) + new_node += blanks + # write new node + self.buckets.write(new_node) + # update old node data + self._update_size( + node_start, old_node_size) + + self._read_single_node_key.delete(node_start) + self._read_node_nr_of_elements_and_children_flag.delete( + node_start) + + return new_node_start, new_key + elif nr_of_keys_to_rewrite > half_size: # insert key into first half of node + # seek for first key to rewrite + self.buckets.seek(self._calculate_key_position(node_start, self.node_capacity - nr_of_keys_to_rewrite, 'n') + + self.pointer_size) + # read all keys with key>new_key + data = self.buckets.read( + nr_of_keys_to_rewrite * (self.key_size + self.pointer_size)) + old_node_data = struct.unpack( + '<' + nr_of_keys_to_rewrite * (self.key_format + self.pointer_format), data) + key_moved_to_parent_node = old_node_data[-( + new_node_size + 1) * 2] + self.buckets.seek(0, 2) + new_node_start = self.buckets.tell() + # prepare new node_data + new_node = struct.pack('<' + self.node_heading_format + + self.pointer_format + (self.key_format + + self.pointer_format) * new_node_size, + new_node_size, + children_flag, + old_node_data[-new_node_size * 2 - 1], + *old_node_data[-new_node_size * 2:]) + new_node += blanks + # write new node + self.buckets.write(new_node) + self._update_size( + node_start, old_node_size) + # seek position of new key in first half + self.buckets.seek(self._calculate_key_position(node_start, self.node_capacity - nr_of_keys_to_rewrite, 'n') + + self.pointer_size) + # write new key and keys after + self.buckets.write( + struct.pack( + '<' + (self.key_format + self.pointer_format) * + (nr_of_keys_to_rewrite - new_node_size), + new_key, + new_pointer, + *old_node_data[:-(new_node_size + 1) * 2])) + + self._read_single_node_key.delete(node_start) + self._read_node_nr_of_elements_and_children_flag.delete( + node_start) + + return new_node_start, key_moved_to_parent_node + else: # key goes into second half + # reading second half of node + self.buckets.seek(self._calculate_key_position(node_start, + old_node_size, + 'n') + + self.pointer_size) + data = self.buckets.read( + new_node_size * (self.key_size + self.pointer_size)) + old_node_data = struct.unpack('<' + new_node_size * + (self.key_format + self.pointer_format), data) + # find key which goes to parent node + key_moved_to_parent_node = old_node_data[0] + self.buckets.seek(0, 2) # end of file + new_node_start = self.buckets.tell() + index_of_records_split = nr_of_keys_to_rewrite * 2 + # prepare new node_data + first_leaf_pointer = old_node_data[1] + old_node_data = old_node_data[2:] + if index_of_records_split: + keys_before = old_node_data[:-index_of_records_split] + keys_after = old_node_data[-index_of_records_split:] + else: + keys_before = old_node_data + keys_after = [] + new_node = struct.pack('<' + self.node_heading_format + self.pointer_format + + (self.key_format + self.pointer_format) * + (new_node_size - + nr_of_keys_to_rewrite - 1), + new_node_size, + children_flag, + first_leaf_pointer, + *keys_before) + new_node += struct.pack('<' + (self.key_format + self.pointer_format) * + (nr_of_keys_to_rewrite + 1), + new_key, + new_pointer, + *keys_after) + new_node += blanks + # write new node + self.buckets.write(new_node) + self._update_size(node_start, old_node_size) + + self._read_single_node_key.delete(node_start) + self._read_node_nr_of_elements_and_children_flag.delete( + node_start) + + return new_node_start, key_moved_to_parent_node + + def insert_first_record_into_leaf(self, leaf_start, key, doc_id, start, size, status): + self.buckets.seek(leaf_start) + self.buckets.write(struct.pack('<' + self.elements_counter_format, + 1)) + self.buckets.seek(leaf_start + self.leaf_heading_size) + self.buckets.write(struct.pack('<' + self.single_leaf_record_format, + key, + doc_id, + start, + size, + status)) + +# self._read_single_leaf_record.delete(leaf_start) + self._find_key_in_leaf.delete(leaf_start) + self._read_leaf_nr_of_elements.delete(leaf_start) + self._read_leaf_nr_of_elements_and_neighbours.delete(leaf_start) + + def _insert_new_record_into_leaf(self, leaf_start, key, doc_id, start, size, status, nodes_stack, indexes): + nr_of_elements = self._read_leaf_nr_of_elements(leaf_start) + if nr_of_elements == 0: + self.insert_first_record_into_leaf( + leaf_start, key, doc_id, start, size, status) + return + leaf_start, new_record_position, nr_of_records_to_rewrite, full_leaf, on_deleted\ + = self._find_place_in_leaf(key, leaf_start, nr_of_elements) + if full_leaf: + try: # check if leaf has parent node + leaf_parent_pointer = nodes_stack.pop() + except IndexError: # leaf is a root + leaf_parent_pointer = 0 + split_data = self._split_leaf(leaf_start, + nr_of_records_to_rewrite, + key, + doc_id, + start, + size, + status, + create_new_root=(False if leaf_parent_pointer else True)) + if split_data is not None: # means that split created new root or replaced split with update_if_has_deleted + new_leaf_start_position, key_moved_to_parent_node = split_data + self._insert_new_key_into_node(leaf_parent_pointer, + key_moved_to_parent_node, + leaf_start, + new_leaf_start_position, + nodes_stack, + indexes) + else: # there is a place for record in leaf + self.buckets.seek(leaf_start) + self._update_leaf( + leaf_start, new_record_position, nr_of_elements, nr_of_records_to_rewrite, + on_deleted, key, doc_id, start, size, status) + + def _update_node(self, new_key_position, nr_of_keys_to_rewrite, new_key, new_pointer): + if nr_of_keys_to_rewrite == 0: + self.buckets.seek(new_key_position) + self.buckets.write( + struct.pack('<' + self.key_format + self.pointer_format, + new_key, + new_pointer)) + self.flush() + else: + self.buckets.seek(new_key_position) + data = self.buckets.read(nr_of_keys_to_rewrite * ( + self.key_size + self.pointer_size)) + keys_to_rewrite = struct.unpack( + '<' + nr_of_keys_to_rewrite * (self.key_format + self.pointer_format), data) + self.buckets.seek(new_key_position) + self.buckets.write( + struct.pack( + '<' + (nr_of_keys_to_rewrite + 1) * + (self.key_format + self.pointer_format), + new_key, + new_pointer, + *keys_to_rewrite)) + self.flush() + + def _insert_new_key_into_node(self, node_start, new_key, old_half_start, new_half_start, nodes_stack, indexes): + parent_key_index = indexes.pop() + nr_of_elements, children_flag = self._read_node_nr_of_elements_and_children_flag(node_start) + parent_prev_pointer = self._read_single_node_key( + node_start, parent_key_index)[0] + if parent_prev_pointer == old_half_start: # splited child was on the left side of his parent key, must write new key before it + new_key_position = self.pointer_size + self._calculate_key_position(node_start, parent_key_index, 'n') + nr_of_keys_to_rewrite = nr_of_elements - parent_key_index + else: # splited child was on the right side of his parent key, must write new key after it + new_key_position = self.pointer_size + self._calculate_key_position(node_start, parent_key_index + 1, 'n') + nr_of_keys_to_rewrite = nr_of_elements - (parent_key_index + 1) + if nr_of_elements == self.node_capacity: + try: # check if node has parent + node_parent_pointer = nodes_stack.pop() + except IndexError: # node is a root + node_parent_pointer = 0 + new_data = self._split_node(node_start, + nr_of_keys_to_rewrite, + new_key, + new_half_start, + children_flag, + create_new_root=(False if node_parent_pointer else True)) + if new_data: # if not new_data, new root has been created + new_node_start_position, key_moved_to_parent_node = new_data + self._insert_new_key_into_node(node_parent_pointer, + key_moved_to_parent_node, + node_start, + new_node_start_position, + nodes_stack, + indexes) + + self._find_first_key_occurence_in_node.delete(node_start) + self._find_last_key_occurence_in_node.delete(node_start) + else: # there is a empty slot for new key in node + self._update_size(node_start, nr_of_elements + 1) + self._update_node(new_key_position, + nr_of_keys_to_rewrite, + new_key, + new_half_start) + + self._find_first_key_occurence_in_node.delete(node_start) + self._find_last_key_occurence_in_node.delete(node_start) + self._read_single_node_key.delete(node_start) + self._read_node_nr_of_elements_and_children_flag.delete(node_start) + + def _find_leaf_to_insert(self, key): + """ + Traverses tree in search for leaf for insert, remembering parent nodes in path, + looks for last occurence of key if already in tree. + """ + nodes_stack = [self.data_start] + if self.root_flag == 'l': + return nodes_stack, [] + else: + nr_of_elements, curr_child_flag = self._read_node_nr_of_elements_and_children_flag(self.data_start) + curr_index, curr_pointer = self._find_last_key_occurence_in_node( + self.data_start, key, nr_of_elements) + nodes_stack.append(curr_pointer) + indexes = [curr_index] + while(curr_child_flag == 'n'): + nr_of_elements, curr_child_flag = self._read_node_nr_of_elements_and_children_flag(curr_pointer) + curr_index, curr_pointer = self._find_last_key_occurence_in_node(curr_pointer, key, nr_of_elements) + nodes_stack.append(curr_pointer) + indexes.append(curr_index) + return nodes_stack, indexes + # nodes stack contains start addreses of nodes directly above leaf with key, indexes match keys adjacent nodes_stack values (as pointers) + # required when inserting new keys in upper tree levels + + def _find_leaf_with_last_key_occurence(self, key): + if self.root_flag == 'l': + return self.data_start + else: + nr_of_elements, curr_child_flag = self._read_node_nr_of_elements_and_children_flag(self.data_start) + curr_position = self._find_last_key_occurence_in_node( + self.data_start, key, nr_of_elements)[1] + while(curr_child_flag == 'n'): + nr_of_elements, curr_child_flag = self._read_node_nr_of_elements_and_children_flag(curr_position) + curr_position = self._find_last_key_occurence_in_node( + curr_position, key, nr_of_elements)[1] + return curr_position + + def _find_leaf_with_first_key_occurence(self, key): + if self.root_flag == 'l': + return self.data_start + else: + nr_of_elements, curr_child_flag = self._read_node_nr_of_elements_and_children_flag(self.data_start) + curr_position = self._find_first_key_occurence_in_node( + self.data_start, key, nr_of_elements)[1] + while(curr_child_flag == 'n'): + nr_of_elements, curr_child_flag = self._read_node_nr_of_elements_and_children_flag(curr_position) + curr_position = self._find_first_key_occurence_in_node( + curr_position, key, nr_of_elements)[1] + return curr_position + + def _find_key(self, key): + containing_leaf_start = self._find_leaf_with_first_key_occurence(key) + nr_of_elements, prev_leaf, next_leaf = self._read_leaf_nr_of_elements_and_neighbours(containing_leaf_start) + try: + doc_id, l_key, start, size, status = self._find_key_in_leaf( + containing_leaf_start, key, nr_of_elements) + except ElemNotFound: + if next_leaf: + nr_of_elements = self._read_leaf_nr_of_elements(next_leaf) + else: + raise ElemNotFound + doc_id, l_key, start, size, status = self._find_key_in_leaf( + next_leaf, key, nr_of_elements) + return doc_id, l_key, start, size, status + + def _find_key_to_update(self, key, doc_id): + """ + Search tree for key that matches not only given key but also doc_id. + """ + containing_leaf_start = self._find_leaf_with_first_key_occurence(key) + nr_of_elements, prev_leaf, next_leaf = self._read_leaf_nr_of_elements_and_neighbours(containing_leaf_start) + try: + leaf_start, record_index, doc_id, l_key, start, size, status = self._find_key_in_leaf_for_update(key, + doc_id, + containing_leaf_start, + nr_of_elements) + except ElemNotFound: + if next_leaf: + nr_of_elements = self._read_leaf_nr_of_elements(next_leaf) + else: + raise TryReindexException() + try: + leaf_start, record_index, doc_id, l_key, start, size, status = self._find_key_in_leaf_for_update(key, + doc_id, + next_leaf, + nr_of_elements) + except ElemNotFound: + raise TryReindexException() + return leaf_start, record_index, doc_id, l_key, start, size, status + + def update(self, doc_id, key, u_start=0, u_size=0, u_status='o'): + containing_leaf_start, element_index, old_doc_id, old_key, old_start, old_size, old_status = self._find_key_to_update(key, doc_id) + new_data = (old_doc_id, old_start, old_size, old_status) + if not u_start: + new_data[1] = u_start + if not u_size: + new_data[2] = u_size + if not u_status: + new_data[3] = u_status + self._update_element(containing_leaf_start, element_index, new_data) + + self._find_key.delete(key) + self._match_doc_id.delete(doc_id) + self._find_key_in_leaf.delete(containing_leaf_start, key) + return True + + def delete(self, doc_id, key, start=0, size=0): + containing_leaf_start, element_index = self._find_key_to_update( + key, doc_id)[:2] + self._delete_element(containing_leaf_start, element_index) + + self._find_key.delete(key) + self._match_doc_id.delete(doc_id) + self._find_key_in_leaf.delete(containing_leaf_start, key) + return True + + def _find_key_many(self, key, limit=1, offset=0): + leaf_with_key = self._find_leaf_with_first_key_occurence(key) + nr_of_elements, prev_leaf, next_leaf = self._read_leaf_nr_of_elements_and_neighbours(leaf_with_key) + try: + leaf_with_key, key_index = self._find_index_of_first_key_equal( + key, leaf_with_key, nr_of_elements) + nr_of_elements, prev_leaf, next_leaf = self._read_leaf_nr_of_elements_and_neighbours(leaf_with_key) + except ElemNotFound: + leaf_with_key = next_leaf + key_index = 0 + nr_of_elements, prev_leaf, next_leaf = self._read_leaf_nr_of_elements_and_neighbours(leaf_with_key) + while offset: + if key_index < nr_of_elements: + curr_key, doc_id, start, size, status = self._read_single_leaf_record( + leaf_with_key, key_index) + if key == curr_key: + if status != 'd': + offset -= 1 + key_index += 1 + else: + return + else: + key_index = 0 + if next_leaf: + leaf_with_key = next_leaf + nr_of_elements, prev_leaf, next_leaf = self._read_leaf_nr_of_elements_and_neighbours(next_leaf) + else: + return + while limit: + if key_index < nr_of_elements: + curr_key, doc_id, start, size, status = self._read_single_leaf_record( + leaf_with_key, key_index) + if key == curr_key: + if status != 'd': + yield doc_id, start, size, status + limit -= 1 + key_index += 1 + else: + return + else: + key_index = 0 + if next_leaf: + leaf_with_key = next_leaf + nr_of_elements, prev_leaf, next_leaf = self._read_leaf_nr_of_elements_and_neighbours(next_leaf) + else: + return + + def _find_key_smaller(self, key, limit=1, offset=0): + leaf_with_key = self._find_leaf_with_first_key_occurence(key) + nr_of_elements, prev_leaf, next_leaf = self._read_leaf_nr_of_elements_and_neighbours(leaf_with_key) + leaf_with_key, key_index = self._find_index_of_first_key_equal_or_smaller_key(key, leaf_with_key, nr_of_elements) + nr_of_elements, prev_leaf, next_leaf = self._read_leaf_nr_of_elements_and_neighbours(leaf_with_key) + curr_key = self._read_single_leaf_record(leaf_with_key, key_index)[0] + if curr_key >= key: + key_index -= 1 + while offset: + if key_index >= 0: + key, doc_id, start, size, status = self._read_single_leaf_record( + leaf_with_key, key_index) + if status != 'd': + offset -= 1 + key_index -= 1 + else: + if prev_leaf: + leaf_with_key = prev_leaf + nr_of_elements, prev_leaf, next_leaf = self._read_leaf_nr_of_elements_and_neighbours(prev_leaf) + key_index = nr_of_elements - 1 + else: + return + while limit: + if key_index >= 0: + key, doc_id, start, size, status = self._read_single_leaf_record( + leaf_with_key, key_index) + if status != 'd': + yield doc_id, key, start, size, status + limit -= 1 + key_index -= 1 + else: + if prev_leaf: + leaf_with_key = prev_leaf + nr_of_elements, prev_leaf, next_leaf = self._read_leaf_nr_of_elements_and_neighbours(prev_leaf) + key_index = nr_of_elements - 1 + else: + return + + def _find_key_equal_and_smaller(self, key, limit=1, offset=0): + leaf_with_key = self._find_leaf_with_last_key_occurence(key) + nr_of_elements, prev_leaf, next_leaf = self._read_leaf_nr_of_elements_and_neighbours(leaf_with_key) + try: + leaf_with_key, key_index = self._find_index_of_last_key_equal_or_smaller_key(key, leaf_with_key, nr_of_elements) + nr_of_elements, prev_leaf, next_leaf = self._read_leaf_nr_of_elements_and_neighbours(leaf_with_key) + except ElemNotFound: + leaf_with_key = prev_leaf + key_index = self._read_leaf_nr_of_elements_and_neighbours( + leaf_with_key)[0] + curr_key = self._read_single_leaf_record(leaf_with_key, key_index)[0] + if curr_key > key: + key_index -= 1 + while offset: + if key_index >= 0: + key, doc_id, start, size, status = self._read_single_leaf_record( + leaf_with_key, key_index) + if status != 'd': + offset -= 1 + key_index -= 1 + else: + if prev_leaf: + leaf_with_key = prev_leaf + nr_of_elements, prev_leaf, next_leaf = self._read_leaf_nr_of_elements_and_neighbours(prev_leaf) + key_index = nr_of_elements - 1 + else: + return + while limit: + if key_index >= 0: + key, doc_id, start, size, status = self._read_single_leaf_record( + leaf_with_key, key_index) + if status != 'd': + yield doc_id, key, start, size, status + limit -= 1 + key_index -= 1 + else: + if prev_leaf: + leaf_with_key = prev_leaf + nr_of_elements, prev_leaf, next_leaf = self._read_leaf_nr_of_elements_and_neighbours(prev_leaf) + key_index = nr_of_elements - 1 + else: + return + + def _find_key_bigger(self, key, limit=1, offset=0): + leaf_with_key = self._find_leaf_with_last_key_occurence(key) + nr_of_elements, prev_leaf, next_leaf = self._read_leaf_nr_of_elements_and_neighbours(leaf_with_key) + try: + leaf_with_key, key_index = self._find_index_of_last_key_equal_or_smaller_key(key, leaf_with_key, nr_of_elements) + nr_of_elements, prev_leaf, next_leaf = self._read_leaf_nr_of_elements_and_neighbours(leaf_with_key) + except ElemNotFound: + key_index = 0 + curr_key = self._read_single_leaf_record(leaf_with_key, key_index)[0] + if curr_key <= key: + key_index += 1 + while offset: + if key_index < nr_of_elements: + curr_key, doc_id, start, size, status = self._read_single_leaf_record( + leaf_with_key, key_index) + if status != 'd': + offset -= 1 + key_index += 1 + else: + key_index = 0 + if next_leaf: + leaf_with_key = next_leaf + nr_of_elements, prev_leaf, next_leaf = self._read_leaf_nr_of_elements_and_neighbours(next_leaf) + else: + return + while limit: + if key_index < nr_of_elements: + curr_key, doc_id, start, size, status = self._read_single_leaf_record( + leaf_with_key, key_index) + if status != 'd': + yield doc_id, curr_key, start, size, status + limit -= 1 + key_index += 1 + else: + key_index = 0 + if next_leaf: + leaf_with_key = next_leaf + nr_of_elements, prev_leaf, next_leaf = self._read_leaf_nr_of_elements_and_neighbours(next_leaf) + else: + return + + def _find_key_equal_and_bigger(self, key, limit=1, offset=0): + leaf_with_key = self._find_leaf_with_first_key_occurence(key) + nr_of_elements, prev_leaf, next_leaf = self._read_leaf_nr_of_elements_and_neighbours(leaf_with_key) + leaf_with_key, key_index = self._find_index_of_first_key_equal_or_smaller_key(key, leaf_with_key, nr_of_elements) + nr_of_elements, prev_leaf, next_leaf = self._read_leaf_nr_of_elements_and_neighbours(leaf_with_key) + curr_key = self._read_single_leaf_record(leaf_with_key, key_index)[0] + if curr_key < key: + key_index += 1 + while offset: + if key_index < nr_of_elements: + curr_key, doc_id, start, size, status = self._read_single_leaf_record( + leaf_with_key, key_index) + if status != 'd': + offset -= 1 + key_index += 1 + else: + key_index = 0 + if next_leaf: + leaf_with_key = next_leaf + nr_of_elements, prev_leaf, next_leaf = self._read_leaf_nr_of_elements_and_neighbours(next_leaf) + else: + return + while limit: + if key_index < nr_of_elements: + curr_key, doc_id, start, size, status = self._read_single_leaf_record( + leaf_with_key, key_index) + if status != 'd': + yield doc_id, curr_key, start, size, status + limit -= 1 + key_index += 1 + else: + key_index = 0 + if next_leaf: + leaf_with_key = next_leaf + nr_of_elements, prev_leaf, next_leaf = self._read_leaf_nr_of_elements_and_neighbours(next_leaf) + else: + return + + def _find_key_between(self, start, end, limit, offset, inclusive_start, inclusive_end): + """ + Returns generator containing all keys withing given interval. + """ + if inclusive_start: + leaf_with_key = self._find_leaf_with_first_key_occurence(start) + nr_of_elements, prev_leaf, next_leaf = self._read_leaf_nr_of_elements_and_neighbours(leaf_with_key) + leaf_with_key, key_index = self._find_index_of_first_key_equal_or_smaller_key(start, leaf_with_key, nr_of_elements) + nr_of_elements, prev_leaf, next_leaf = self._read_leaf_nr_of_elements_and_neighbours(leaf_with_key) + curr_key = self._read_single_leaf_record( + leaf_with_key, key_index)[0] + if curr_key < start: + key_index += 1 + else: + leaf_with_key = self._find_leaf_with_last_key_occurence(start) + nr_of_elements, prev_leaf, next_leaf = self._read_leaf_nr_of_elements_and_neighbours(leaf_with_key) + leaf_with_key, key_index = self._find_index_of_last_key_equal_or_smaller_key(start, leaf_with_key, nr_of_elements) + curr_key, curr_doc_id, curr_start, curr_size, curr_status = self._read_single_leaf_record(leaf_with_key, key_index) + if curr_key <= start: + key_index += 1 + while offset: + if key_index < nr_of_elements: + curr_key, curr_doc_id, curr_start, curr_size, curr_status = self._read_single_leaf_record(leaf_with_key, key_index) + if curr_status != 'd': + offset -= 1 + key_index += 1 + else: + key_index = 0 + if next_leaf: + leaf_with_key = next_leaf + nr_of_elements, prev_leaf, next_leaf = self._read_leaf_nr_of_elements_and_neighbours(next_leaf) + else: + return + while limit: + if key_index < nr_of_elements: + curr_key, curr_doc_id, curr_start, curr_size, curr_status = self._read_single_leaf_record(leaf_with_key, key_index) + if curr_key > end or (curr_key == end and not inclusive_end): + return + elif curr_status != 'd': + yield curr_doc_id, curr_key, curr_start, curr_size, curr_status + limit -= 1 + key_index += 1 + else: + key_index = 0 + if next_leaf: + leaf_with_key = next_leaf + nr_of_elements, prev_leaf, next_leaf = self._read_leaf_nr_of_elements_and_neighbours(next_leaf) + else: + return + + def get(self, key): + return self._find_key(self.make_key(key)) + + def get_many(self, key, limit=1, offset=0): + return self._find_key_many(self.make_key(key), limit, offset) + + def get_between(self, start, end, limit=1, offset=0, inclusive_start=True, inclusive_end=True): + if start is None: + end = self.make_key(end) + if inclusive_end: + return self._find_key_equal_and_smaller(end, limit, offset) + else: + return self._find_key_smaller(end, limit, offset) + elif end is None: + start = self.make_key(start) + if inclusive_start: + return self._find_key_equal_and_bigger(start, limit, offset) + else: + return self._find_key_bigger(start, limit, offset) + else: + start = self.make_key(start) + end = self.make_key(end) + return self._find_key_between(start, end, limit, offset, inclusive_start, inclusive_end) + + def all(self, limit=-1, offset=0): + """ + Traverses linked list of all tree leaves and returns generator containing all elements stored in index. + """ + if self.root_flag == 'n': + leaf_start = self.data_start + self.node_size + else: + leaf_start = self.data_start + nr_of_elements, prev_leaf, next_leaf = self._read_leaf_nr_of_elements_and_neighbours(leaf_start) + key_index = 0 + while offset: + if key_index < nr_of_elements: + curr_key, doc_id, start, size, status = self._read_single_leaf_record( + leaf_start, key_index) + if status != 'd': + offset -= 1 + key_index += 1 + else: + key_index = 0 + if next_leaf: + leaf_start = next_leaf + nr_of_elements, prev_leaf, next_leaf = self._read_leaf_nr_of_elements_and_neighbours(next_leaf) + else: + return + while limit: + if key_index < nr_of_elements: + curr_key, doc_id, start, size, status = self._read_single_leaf_record( + leaf_start, key_index) + if status != 'd': + yield doc_id, curr_key, start, size, status + limit -= 1 + key_index += 1 + else: + key_index = 0 + if next_leaf: + leaf_start = next_leaf + nr_of_elements, prev_leaf, next_leaf = self._read_leaf_nr_of_elements_and_neighbours(next_leaf) + else: + return + + def make_key(self, key): + raise NotImplementedError() + + def make_key_value(self, data): + raise NotImplementedError() + + def _open_storage(self): + s = globals()[self.storage_class] + if not self.storage: + self.storage = s(self.db_path, self.name) + self.storage.open() + + def _create_storage(self): + s = globals()[self.storage_class] + if not self.storage: + self.storage = s(self.db_path, self.name) + self.storage.create() + + def compact(self, node_capacity=0): + if not node_capacity: + node_capacity = self.node_capacity + + compact_ind = self.__class__( + self.db_path, self.name + '_compact', node_capacity=node_capacity) + compact_ind.create_index() + + gen = self.all() + while True: + try: + doc_id, key, start, size, status = gen.next() + except StopIteration: + break + self.storage._f.seek(start) + value = self.storage._f.read(size) + start_ = compact_ind.storage._f.tell() + compact_ind.storage._f.write(value) + compact_ind.insert(doc_id, key, start_, size, status) + + compact_ind.close_index() + original_name = self.name + # os.unlink(os.path.join(self.db_path, self.name + "_buck")) + self.close_index() + shutil.move(os.path.join(compact_ind.db_path, compact_ind. + name + "_buck"), os.path.join(self.db_path, self.name + "_buck")) + shutil.move(os.path.join(compact_ind.db_path, compact_ind. + name + "_stor"), os.path.join(self.db_path, self.name + "_stor")) + # self.name = original_name + self.open_index() # reload... + self.name = original_name + self._save_params(dict(name=original_name)) + self._fix_params() + self._clear_cache() + return True + + def _fix_params(self): + super(IU_TreeBasedIndex, self)._fix_params() + self._count_props() + + def _clear_cache(self): + self._find_key.clear() + self._match_doc_id.clear() +# self._read_single_leaf_record.clear() + self._find_key_in_leaf.clear() + self._read_single_node_key.clear() + self._find_first_key_occurence_in_node.clear() + self._find_last_key_occurence_in_node.clear() + self._read_leaf_nr_of_elements.clear() + self._read_leaf_neighbours.clear() + self._read_leaf_nr_of_elements_and_neighbours.clear() + self._read_node_nr_of_elements_and_children_flag.clear() + + def close_index(self): + super(IU_TreeBasedIndex, self).close_index() + self._clear_cache() + + +class IU_MultiTreeBasedIndex(IU_TreeBasedIndex): + """ + Class that allows to index more than one key per database record. + + It operates very well on GET/INSERT. It's not optimized for + UPDATE operations (will always readd everything) + """ + + def __init__(self, *args, **kwargs): + super(IU_MultiTreeBasedIndex, self).__init__(*args, **kwargs) + + def insert(self, doc_id, key, start, size, status='o'): + if isinstance(key, (list, tuple)): + key = set(key) + elif not isinstance(key, set): + key = set([key]) + ins = super(IU_MultiTreeBasedIndex, self).insert + for curr_key in key: + ins(doc_id, curr_key, start, size, status) + return True + + def update(self, doc_id, key, u_start, u_size, u_status='o'): + if isinstance(key, (list, tuple)): + key = set(key) + elif not isinstance(key, set): + key = set([key]) + upd = super(IU_MultiTreeBasedIndex, self).update + for curr_key in key: + upd(doc_id, curr_key, u_start, u_size, u_status) + + def delete(self, doc_id, key, start=0, size=0): + if isinstance(key, (list, tuple)): + key = set(key) + elif not isinstance(key, set): + key = set([key]) + delete = super(IU_MultiTreeBasedIndex, self).delete + for curr_key in key: + delete(doc_id, curr_key, start, size) + + def get(self, key): + return super(IU_MultiTreeBasedIndex, self).get(key) + + def make_key_value(self, data): + raise NotImplementedError() + + +# classes for public use, done in this way because of +# generation static files with indexes (_index directory) + + +class TreeBasedIndex(IU_TreeBasedIndex): + pass + + +class MultiTreeBasedIndex(IU_MultiTreeBasedIndex): + """ + It allows to index more than one key for record. (ie. prefix/infix/suffix search mechanizms) + That class is designed to be used in custom indexes. + """ + pass