diff --git a/libs/CodernityDB3/__init__.py b/libs/CodernityDB3/__init__.py new file mode 100644 index 00000000..8399a60f --- /dev/null +++ b/libs/CodernityDB3/__init__.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright 2011-2013 Codernity (http://codernity.com) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +__version__ = '0.4.2' +__license__ = "Apache 2.0" diff --git a/libs/CodernityDB3/database.py b/libs/CodernityDB3/database.py new file mode 100644 index 00000000..f3493128 --- /dev/null +++ b/libs/CodernityDB3/database.py @@ -0,0 +1,1218 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright 2011-2013 Codernity (http://codernity.com) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +bytes = str + +import os +import io +from inspect import getsource + +# for custom indexes +from CodernityDB3.storage import Storage, IU_Storage +from CodernityDB3.hash_index import (IU_UniqueHashIndex, + IU_HashIndex, + HashIndex, + UniqueHashIndex) +# normal imports + +from CodernityDB3.index import (ElemNotFound, + DocIdNotFound, + IndexException, + Index, + TryReindexException, + ReindexException, + IndexNotFoundException, + IndexConflict) + +from CodernityDB3.misc import NONE + +from CodernityDB3.env import cdb_environment + +from random import randrange + +import warnings +import collections + + +def header_for_indexes(index_name, index_class, db_custom="", ind_custom="", classes_code=""): + return """# %s +# %s + +# inserted automatically +import os +import marshal + +import struct +import shutil + +from hashlib import md5 + +# custom db code start +# db_custom +%s + +# custom index code start +# ind_custom +%s + +# source of classes in index.classes_code +# classes_code +%s + +# index code start + +""" % (index_name, index_class, db_custom, ind_custom, classes_code) + + +class DatabaseException(Exception): + pass + + +class PreconditionsException(DatabaseException): + pass + + +class RecordDeleted(DatabaseException): + pass + + +class RecordNotFound(DatabaseException): + pass + + +class RevConflict(DatabaseException): + pass + + +class DatabaseConflict(DatabaseException): + pass + + +class DatabasePathException(DatabaseException): + pass + + +class DatabaseIsNotOpened(PreconditionsException): + pass + + +class Database(object): + """ + A default single thread database object. + """ + + custom_header = "" # : use it for imports required by your database + + def __init__(self, path): + self.path = path + self.storage = None + self.indexes = [] + self.id_ind = None + self.indexes_names = {} + self.opened = False + + def create_new_rev(self, old_rev=None): + """ + Creates new revision number based on previous one. + Increments it + random bytes. On overflow starts from 0 again. + """ + if old_rev: + try: + rev_num = int(old_rev[:4], 16) + except: + raise RevConflict() + rev_num += 1 + if rev_num > 65025: + # starting the counter from 0 again + rev_num = 0 + rnd = randrange(65536) + return "%04x%04x" % (rev_num, rnd) + else: + # new rev + rnd = randrange(256 ** 2) + return '0001%04x' % rnd + + def __not_opened(self): + if not self.opened: + raise DatabaseIsNotOpened("Database is not opened") + + def set_indexes(self, indexes=[]): + """ + Set indexes using ``indexes`` param + + :param indexes: indexes to set in db + :type indexes: iterable of :py:class:`CodernityDB3.index.Index` objects. + + """ + for ind in indexes: + self.add_index(ind, create=False) + + def _add_single_index(self, p, i, index): + """ + Adds single index to a database. + It will use :py:meth:`inspect.getsource` to get class source. + Then it will build real index file, save it in ``_indexes`` directory. + """ + code = getsource(index.__class__) + if not code.startswith('c'): # fix for indented index codes + import textwrap + code = textwrap.dedent(code) + index._order = i + cls_code = getattr(index, 'classes_code', []) + classes_code = "" + for curr in cls_code: + classes_code += getsource(curr) + '\n\n' + with io.FileIO(os.path.join(p, "%.2d%s" % (i, index.name) + '.py'), 'w') as f: + f.write(str.encode(header_for_indexes(index.name, + index.__class__.__name__, + getattr(self, 'custom_header', ''), + getattr(index, 'custom_header', ''), + classes_code))) + f.write(str.encode(code)) + return True + + def _read_index_single(self, p, ind, ind_kwargs={}): + """ + It will read single index from index file (ie. generated in :py:meth:`._add_single_index`). + Then it will perform ``exec`` on that code + + If error will occur the index file will be saved with ``_broken`` suffix + + :param p: path + :param ind: index name (will be joined with *p*) + :returns: new index object + """ + with io.FileIO(os.path.join(p, ind), 'r') as f: + name = f.readline()[2:].strip() + _class = f.readline()[2:].strip() + code = f.read() + try: + obj = compile(code, '', f.__name__, repr(args[1:]) + res = f(*args, **kwargs) +# if db.opened: +# db.flush() +# print '<=', f.__name__, repr(args[1:]) + return res + return _inner + + def __new__(cls, classname, bases, attr): + new_attr = {} + for base in bases: + for b_attr in dir(base): + a = getattr(base, b_attr, None) + if isinstance(a, MethodType) and not b_attr.startswith('_'): + if b_attr == 'flush' or b_attr == 'flush_indexes': + pass + else: + # setattr(base, b_attr, SuperLock.wrapper(a)) + new_attr[b_attr] = SuperLock.wrapper(a) + for attr_name, attr_value in attr.items(): + if isinstance(attr_value, FunctionType) and not attr_name.startswith('_'): + attr_value = SuperLock.wrapper(attr_value) + new_attr[attr_name] = attr_value + new_attr['super_lock'] = RLock() + return type.__new__(cls, classname, bases, new_attr) + + +class SuperThreadSafeDatabase(Database, metaclass=SuperLock): + """ + Thread safe version that always allows single thread to use db. + It adds the same lock for all methods, so only one operation can be + performed in given time. Completely different implementation + than ThreadSafe version (without super word) + """ + + def __init__(self, *args, **kwargs): + super(SuperThreadSafeDatabase, self).__init__(*args, **kwargs) + + def __patch_index_gens(self, name): + ind = self.indexes_names[name] + for c in ('all', 'get_many'): + m = getattr(ind, c) + if getattr(ind, c + "_orig", None): + return + m_fixed = th_safe_gen.wrapper(m, name, c, self.super_lock) + setattr(ind, c, m_fixed) + setattr(ind, c + '_orig', m) + + def open(self, *args, **kwargs): + res = super(SuperThreadSafeDatabase, self).open(*args, **kwargs) + for name in self.indexes_names.keys(): + self.__patch_index_gens(name) + return res + + def create(self, *args, **kwargs): + res = super(SuperThreadSafeDatabase, self).create(*args, **kwargs) + for name in self.indexes_names.keys(): + self.__patch_index_gens(name) + return res + + def add_index(self, *args, **kwargs): + res = super(SuperThreadSafeDatabase, self).add_index(*args, **kwargs) + self.__patch_index_gens(res) + return res + + def edit_index(self, *args, **kwargs): + res = super(SuperThreadSafeDatabase, self).edit_index(*args, **kwargs) + self.__patch_index_gens(res) + return res diff --git a/libs/CodernityDB3/database_thread_safe.py b/libs/CodernityDB3/database_thread_safe.py new file mode 100644 index 00000000..e18e6425 --- /dev/null +++ b/libs/CodernityDB3/database_thread_safe.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright 2011-2013 Codernity (http://codernity.com) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from threading import RLock + +from CodernityDB3.env import cdb_environment + +cdb_environment['mode'] = "threads" +cdb_environment['rlock_obj'] = RLock + + +from .database_safe_shared import SafeDatabase + + +class ThreadSafeDatabase(SafeDatabase): + """ + Thread safe version of CodernityDB that uses several lock objects, + on different methods / different indexes etc. It's completely different + implementation of locking than SuperThreadSafe one. + """ + pass diff --git a/libs/CodernityDB3/debug_stuff.py b/libs/CodernityDB3/debug_stuff.py new file mode 100644 index 00000000..de03a7db --- /dev/null +++ b/libs/CodernityDB3/debug_stuff.py @@ -0,0 +1,211 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright 2011-2013 Codernity (http://codernity.com) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from CodernityDB3.tree_index import TreeBasedIndex +import struct +import os + +import inspect +from functools import wraps +import json + + +class DebugTreeBasedIndex(TreeBasedIndex): + + def __init__(self, *args, **kwargs): + super(DebugTreeBasedIndex, self).__init__(*args, **kwargs) + + def print_tree(self): + print '-----CURRENT TREE-----' + print self.root_flag + + if self.root_flag == 'l': + print '---ROOT---' + self._print_leaf_data(self.data_start) + return + else: + print '---ROOT---' + self._print_node_data(self.data_start) + nr_of_el, children_flag = self._read_node_nr_of_elements_and_children_flag( + self.data_start) + nodes = [] + for index in range(nr_of_el): + l_pointer, key, r_pointer = self._read_single_node_key( + self.data_start, index) + nodes.append(l_pointer) + nodes.append(r_pointer) + print 'ROOT NODES', nodes + while children_flag == 'n': + self._print_level(nodes, 'n') + new_nodes = [] + for node in nodes: + nr_of_el, children_flag = \ + self._read_node_nr_of_elements_and_children_flag(node) + for index in range(nr_of_el): + l_pointer, key, r_pointer = self._read_single_node_key( + node, index) + new_nodes.append(l_pointer) + new_nodes.append(r_pointer) + nodes = new_nodes + self._print_level(nodes, 'l') + + def _print_level(self, nodes, flag): + print '---NEXT LVL---' + if flag == 'n': + for node in nodes: + self._print_node_data(node) + elif flag == 'l': + for node in nodes: + self._print_leaf_data(node) + + def _print_leaf_data(self, leaf_start_position): + print 'printing data of leaf at', leaf_start_position + nr_of_elements = self._read_leaf_nr_of_elements(leaf_start_position) + self.buckets.seek(leaf_start_position) + data = self.buckets.read(self.leaf_heading_size + + nr_of_elements * self.single_leaf_record_size) + leaf = struct.unpack('<' + self.leaf_heading_format + + nr_of_elements * self.single_leaf_record_format, data) + print leaf + print + + def _print_node_data(self, node_start_position): + print 'printing data of node at', node_start_position + nr_of_elements = self._read_node_nr_of_elements_and_children_flag( + node_start_position)[0] + self.buckets.seek(node_start_position) + data = self.buckets.read(self.node_heading_size + self.pointer_size + + nr_of_elements * (self.key_size + self.pointer_size)) + node = struct.unpack('<' + self.node_heading_format + self.pointer_format + + nr_of_elements * ( + self.key_format + self.pointer_format), + data) + print node + print +# ------------------> + + +def database_step_by_step(db_obj, path=None): + + if not path: + # ugly for multiplatform support.... + p = db_obj.path + p1 = os.path.split(p) + p2 = os.path.split(p1[0]) + p3 = '_'.join([p2[1], 'operation_logger.log']) + path = os.path.join(os.path.split(p2[0])[0], p3) + f_obj = open(path, 'wb') + + __stack = [] # inspect.stack() is not working on pytest etc + + def remove_from_stack(name): + for i in range(len(__stack)): + if __stack[-i] == name: + __stack.pop(-i) + + def __dumper(f): + @wraps(f) + def __inner(*args, **kwargs): + funct_name = f.__name__ + if funct_name == 'count': + name = args[0].__name__ + meth_args = (name,) + args[1:] + elif funct_name in ('reindex_index', 'compact_index'): + name = args[0].name + meth_args = (name,) + args[1:] + else: + meth_args = args + kwargs_copy = kwargs.copy() + res = None + __stack.append(funct_name) + if funct_name == 'insert': + try: + res = f(*args, **kwargs) + except: + packed = json.dumps((funct_name, + meth_args, kwargs_copy, None)) + f_obj.write('%s\n' % packed) + f_obj.flush() + raise + else: + packed = json.dumps((funct_name, + meth_args, kwargs_copy, res)) + f_obj.write('%s\n' % packed) + f_obj.flush() + else: + if funct_name == 'get': + for curr in __stack: + if ('delete' in curr or 'update' in curr) and not curr.startswith('test'): + remove_from_stack(funct_name) + return f(*args, **kwargs) + packed = json.dumps((funct_name, meth_args, kwargs_copy)) + f_obj.write('%s\n' % packed) + f_obj.flush() + res = f(*args, **kwargs) + remove_from_stack(funct_name) + return res + return __inner + + for meth_name, meth_f in inspect.getmembers(db_obj, predicate=inspect.ismethod): + if not meth_name.startswith('_'): + setattr(db_obj, meth_name, __dumper(meth_f)) + + setattr(db_obj, 'operation_logger', f_obj) + + +def database_from_steps(db_obj, path): + # db_obj.insert=lambda data : insert_for_debug(db_obj, data) + with open(path, 'rb') as f_obj: + for current in f_obj: + line = json.loads(current[:-1]) + if line[0] == 'count': + obj = getattr(db_obj, line[1][0]) + line[1] = [obj] + line[1][1:] + name = line[0] + if name == 'insert': + try: + line[1][0].pop('_rev') + except: + pass + elif name in ('delete', 'update'): + el = db_obj.get('id', line[1][0]['_id']) + line[1][0]['_rev'] = el['_rev'] +# print 'FROM STEPS doing', line + meth = getattr(db_obj, line[0], None) + if not meth: + raise Exception("Method = `%s` not found" % line[0]) + + meth(*line[1], **line[2]) + + +# def insert_for_debug(self, data): +# +# _rev = data['_rev'] +# +# if not '_id' in data: +# _id = uuid4().hex +# else: +# _id = data['_id'] +# data['_id'] = _id +# try: +# _id = bytes(_id) +# except: +# raise DatabaseException("`_id` must be valid bytes object") +# self._insert_indexes(_id, _rev, data) +# ret = {'_id': _id, '_rev': _rev} +# data.update(ret) +# return ret diff --git a/libs/CodernityDB3/env.py b/libs/CodernityDB3/env.py new file mode 100644 index 00000000..69ca8cdd --- /dev/null +++ b/libs/CodernityDB3/env.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright 2011-2013 Codernity (http://codernity.com) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +It's CodernityDB environment. +Handles internal informations.' +""" + +cdb_environment = { + 'mode': 'normal' +} diff --git a/libs/CodernityDB3/hash_index.py b/libs/CodernityDB3/hash_index.py new file mode 100644 index 00000000..e55f9b1e --- /dev/null +++ b/libs/CodernityDB3/hash_index.py @@ -0,0 +1,995 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright 2011-2013 Codernity (http://codernity.com) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from CodernityDB3.index import (Index, + IndexException, + DocIdNotFound, + ElemNotFound, + TryReindexException, + IndexPreconditionsException) + +import os +import marshal +import io +import struct +import shutil + +from CodernityDB3.storage import IU_Storage, DummyStorage + +from CodernityDB3.env import cdb_environment + +if cdb_environment.get('rlock_obj'): + from CodernityDB3 import patch + patch.patch_cache_rr(cdb_environment['rlock_obj']) + +from CodernityDB3.rr_cache import cache1lvl + + +from CodernityDB3.misc import random_hex_32 + +try: + from CodernityDB3 import __version__ +except ImportError: + from .__init__ import __version__ + + +class IU_HashIndex(Index): + """ + That class is for Internal Use only, if you want to use HashIndex just subclass the :py:class:`HashIndex` instead this one. + + That design is because main index logic should be always in database not in custom user indexes. + """ + + def __init__(self, db_path, name, entry_line_format='<32s{key}IIcI', hash_lim=0xfffff, storage_class=None, key_format='c'): + """ + The index is capable to solve conflicts by `Separate chaining` + :param db_path: database path + :type db_path: string + :param name: index name + :type name: ascii string + :param line_format: line format, `key_format` parameter value will replace `{key}` if present. + :type line_format: string (32s{key}IIcI by default) {doc_id}{hash_key}{start}{size}{status}{next} + :param hash_lim: maximum hash functon results (remember about birthday problem) count from 0 + :type hash_lim: integer + :param storage_class: Storage class by default it will open standard :py:class:`CodernityDB3.storage.Storage` (if string has to be accesible by globals()[storage_class]) + :type storage_class: class name which will be instance of CodernityDB3.storage.Storage instance or None + :param key_format: a index key format + """ + # Fix types + if isinstance(db_path, str): + db_path = db_path.encode() + if isinstance(name, str): + name = name.encode() + + if key_format and '{key}' in entry_line_format: + entry_line_format = entry_line_format.replace('{key}', key_format) + super(IU_HashIndex, self).__init__(db_path, name) + self.hash_lim = hash_lim + if not storage_class: + storage_class = IU_Storage + if storage_class and not isinstance(storage_class, str): + storage_class = storage_class.__name__ + self.storage_class = storage_class + self.storage = None + + self.bucket_line_format = "= self.data_start: + self.buckets.seek(pos_prev) + data = self.buckets.read(self.entry_line_size) + if data: + doc_id, l_key, start, size, status, _next = self.entry_struct.unpack(data) + self.buckets.seek(pos_prev) + self.buckets.write(self.entry_struct.pack(doc_id, + l_key, + start, + size, + status, + pos_next)) + self.flush() + if pos_next: + self.buckets.seek(pos_next) + data = self.buckets.read(self.entry_line_size) + if data: + doc_id, l_key, start, size, status, _next = self.entry_struct.unpack(data) + self.buckets.seek(pos_next) + self.buckets.write(self.entry_struct.pack(doc_id, + l_key, + start, + size, + status, + _next)) + self.flush() + return + + def delete(self, doc_id, key, start=0, size=0): + # Fix types + if isinstance(doc_id, str): + doc_id = doc_id.encode() + if isinstance(key, str): + key = key.encode() + + start_position = self._calculate_position(key) + self.buckets.seek(start_position) + curr_data = self.buckets.read(self.bucket_line_size) + if curr_data: + location = self.bucket_struct.unpack(curr_data)[0] + else: + # case happens when trying to delete element with new index key in data + # after adding new index to database without reindex + raise TryReindexException() + found_at, _doc_id, _key, start, size, status, _next = self._locate_doc_id(doc_id, key, location) + self.buckets.seek(found_at) + self.buckets.write(self.entry_struct.pack(doc_id, + key, + start, + size, + 'd', + _next)) + self.flush() + # self._fix_link(_key, _prev, _next) + self._find_key.delete(key) + self._locate_doc_id.delete(doc_id) + return True + + def compact(self, hash_lim=None): + + if not hash_lim: + hash_lim = self.hash_lim + + compact_ind = self.__class__( + self.db_path, self.name + '_compact', hash_lim=hash_lim) + compact_ind.create_index() + + gen = self.all() + while True: + try: + doc_id, key, start, size, status = next(gen) + except StopIteration: + break + self.storage._f.seek(start) + value = self.storage._f.read(size) + start_ = compact_ind.storage._f.tell() + compact_ind.storage._f.write(value) + compact_ind.insert(doc_id, key, start_, size, status) + + compact_ind.close_index() + original_name = self.name + # os.unlink(os.path.join(self.db_path, self.name + "_buck")) + self.close_index() + shutil.move(os.path.join(compact_ind.db_path, compact_ind. + name + "_buck"), os.path.join(self.db_path, self.name + "_buck")) + shutil.move(os.path.join(compact_ind.db_path, compact_ind. + name + "_stor"), os.path.join(self.db_path, self.name + "_stor")) + # self.name = original_name + self.open_index() # reload... + self.name = original_name.decode() + self._save_params(dict(name=original_name)) + self._fix_params() + self._clear_cache() + return True + + def make_key(self, key): + return key + + def make_key_value(self, data): + return '1', data + + def _clear_cache(self): + self._find_key.clear() + self._locate_doc_id.clear() + + def close_index(self): + super(IU_HashIndex, self).close_index() + self._clear_cache() + + +class IU_UniqueHashIndex(IU_HashIndex): + """ + Index for *unique* keys! Designed to be a **id** index. + + That class is for Internal Use only, if you want to use UniqueHashIndex just subclass the :py:class:`UniqueHashIndex` instead this one. + + That design is because main index logic should be always in database not in custom user indexes. + """ + + def __init__(self, db_path, name, entry_line_format="<32s8sIIcI", *args, **kwargs): + # Fix types + if isinstance(db_path, str): + db_path = db_path.encode() + if isinstance(name, str): + name = name.encode() + + if 'key' in kwargs: + raise IndexPreconditionsException( + "UniqueHashIndex doesn't accept key parameter'") + super(IU_UniqueHashIndex, self).__init__(db_path, name, + entry_line_format, *args, **kwargs) + self.create_key = random_hex_32 # : set the function to create random key when no _id given + # self.entry_struct=struct.Struct(entry_line_format) + +# @lfu_cache(100) + def _find_key(self, key): + """ + Find the key position + + :param key: the key to find + """ + # Fix types + if isinstance(key, str): + key = key.encode() + + start_position = self._calculate_position(key) + self.seek = self.buckets.seek(start_position) + curr_data = self.buckets.read(self.bucket_line_size) + if curr_data: + location = self.bucket_struct.unpack(curr_data)[0] + found_at, l_key, rev, start, size, status, _next = self._locate_key( + key, location) + + # Fix types + if isinstance(l_key, bytes): + l_key = l_key.decode() + if isinstance(rev, bytes): + rev = rev.decode() + if isinstance(status, bytes): + status = status.decode() + + return l_key, rev, start, size, status + else: + return None, None, 0, 0, 'u' + + def _find_key_many(self, *args, **kwargs): + raise NotImplementedError() + + def _find_place(self, start, key): + """ + Find a place to where put the key. It will iterate using `next` field in record, until + empty `next` found + + :param start: position to start from + """ + # Fix types + if isinstance(key, str): + key = key.encode() + + location = start + while True: + self.buckets.seek(location) + data = self.buckets.read(self.entry_line_size) + # todo, maybe partial read there... + l_key, rev, start, size, status, _next = self.entry_struct.unpack( + data) + if l_key == key: + raise IndexException("The '%s' key already exists" % key) + if not _next or status == 'd': + return self.buckets.tell() - self.entry_line_size, l_key, rev, start, size, status, _next + else: + location = _next # go to next record + + # @lfu_cache(100) + def _locate_key(self, key, start): + """ + Locate position of the key, it will iterate using `next` field in record + until required key will be find. + + :param key: the key to locate + :param start: position to start from + """ + # Fix types + if isinstance(key, str): + key = key.encode() + + location = start + while True: + self.buckets.seek(location) + data = self.buckets.read(self.entry_line_size) + # todo, maybe partial read there... + try: + l_key, rev, start, size, status, _next = self.entry_struct.unpack(data) + except struct.error: + raise ElemNotFound("Location '%s' not found" % key) + if l_key == key: + break + else: + if not _next: + # not found + raise ElemNotFound("Location '%s' not found" % key) + else: + location = _next # go to next record + return self.buckets.tell() - self.entry_line_size, l_key, rev, start, size, status, _next + + def update(self, key, rev, u_start=0, u_size=0, u_status='o'): + # Fix types + if isinstance(key, str): + key = key.encode() + if isinstance(rev, str): + rev = rev.encode() + if isinstance(u_status, str): + u_status = u_status.encode() + + start_position = self._calculate_position(key) + self.buckets.seek(start_position) + curr_data = self.buckets.read(self.bucket_line_size) + # test if it's unique or not really unique hash + + if curr_data: + location = self.bucket_struct.unpack(curr_data)[0] + else: + raise ElemNotFound("Location '%s' not found" % key) + found_at, _key, _rev, start, size, status, _next = self._locate_key( + key, location) + if u_start == 0: + u_start = start + if u_size == 0: + u_size = size + self.buckets.seek(found_at) + self.buckets.write(self.entry_struct.pack(key, + rev, + u_start, + u_size, + u_status, + _next)) + self.flush() + self._find_key.delete(key) + return True + + def insert(self, key, rev, start, size, status='o'): + # Fix types + if isinstance(key, str): + key = key.encode() + if isinstance(rev, str): + rev = rev.encode() + if isinstance(status, str): + status = status.encode() + + start_position = self._calculate_position(key) + self.buckets.seek(start_position) + curr_data = self.buckets.read(self.bucket_line_size) + + # conflict occurs? + if curr_data: + location = self.bucket_struct.unpack(curr_data)[0] + else: + location = 0 + if location: + # last key with that hash + found_at, _key, _rev, _start, _size, _status, _next = self._find_place( + location, key) + self.buckets.seek(0, 2) + wrote_at = self.buckets.tell() + + # check if position is bigger than all hash entries... + if wrote_at < self.data_start: + self.buckets.seek(self.data_start) + wrote_at = self.buckets.tell() + + self.buckets.write(self.entry_struct.pack(key, + rev, + start, + size, + status, + _next)) + +# self.flush() + self.buckets.seek(found_at) + self.buckets.write(self.entry_struct.pack(_key, + _rev, + _start, + _size, + _status, + wrote_at)) + self.flush() + self._find_key.delete(_key) + # self._locate_key.delete(_key) + return True + # raise NotImplementedError + else: + self.buckets.seek(0, 2) + wrote_at = self.buckets.tell() + + # check if position is bigger than all hash entries... + if wrote_at < self.data_start: + self.buckets.seek(self.data_start) + wrote_at = self.buckets.tell() + + self.buckets.write(self.entry_struct.pack(key, + rev, + start, + size, + status, + 0)) + + self.buckets.seek(start_position) + self.buckets.write(self.bucket_struct.pack(wrote_at)) + self.flush() + self._find_key.delete(key) + return True + + def all(self, limit=-1, offset=0): + self.buckets.seek(self.data_start) + while offset: + curr_data = self.buckets.read(self.entry_line_size) + if not curr_data: + break + try: + doc_id, rev, start, size, status, next = self.entry_struct.unpack(curr_data) + except IndexException: + break + else: + if status != 'd': + offset -= 1 + + while limit: + curr_data = self.buckets.read(self.entry_line_size) + if not curr_data: + break + try: + doc_id, rev, start, size, status, next = self.entry_struct.unpack(curr_data) + except IndexException: + break + else: + if status != 'd': + yield doc_id, rev, start, size, status + limit -= 1 + + def get_many(self, *args, **kwargs): + raise NotImplementedError() + + def delete(self, key, start=0, size=0): + # Fix types + if isinstance(key, str): + key = key.encode() + + self.update(key, '00000000', start, size, 'd') + + def make_key_value(self, data): + _id = data['_id'] + try: + _id = data['_id'].encode() + except: + raise IndexPreconditionsException( + "_id must be valid string/bytes object") + if len(_id) != 32: + raise IndexPreconditionsException("Invalid _id lenght") + del data['_id'] + del data['_rev'] + return _id, data + + def destroy(self): + Index.destroy(self) + self._clear_cache() + + def _clear_cache(self): + self._find_key.clear() + + def insert_with_storage(self, _id, _rev, value): + # Fix types + if isinstance(_id, str): + _id = _id.encode() + if isinstance(_rev, str): + _rev = _rev.encode() + if value: + start, size = self.storage.insert(value) + else: + start = 1 + size = 0 + return self.insert(_id, _rev, start, size) + + def update_with_storage(self, _id, _rev, value): + # Fix types + if isinstance(_id, str): + _id = _id.encode() + if isinstance(_rev, str): + _rev = _rev.encode() + + if value: + start, size = self.storage.insert(value) + else: + start = 1 + size = 0 + return self.update(_id, _rev, start, size) + + +class DummyHashIndex(IU_HashIndex): + def __init__(self, db_path, name, entry_line_format="<32s4sIIcI", *args, **kwargs): + super(DummyHashIndex, self).__init__(db_path, name, + entry_line_format, *args, **kwargs) + self.create_key = random_hex_32 # : set the function to create random key when no _id given + # self.entry_struct=struct.Struct(entry_line_format) + + def update(self, *args, **kwargs): + return True + + def insert(self, *args, **kwargs): + return True + + def all(self, *args, **kwargs): + raise StopIteration + + def get(self, *args, **kwargs): + raise ElemNotFound + + def get_many(self, *args, **kwargs): + raise StopIteration + + def delete(self, *args, **kwargs): + pass + + def make_key_value(self, data): + return '1', {'_': 1} + + def destroy(self): + pass + + def _clear_cache(self): + pass + + def _open_storage(self): + if not self.storage: + self.storage = DummyStorage() + self.storage.open() + + def _create_storage(self): + if not self.storage: + self.storage = DummyStorage() + self.storage.create() + + +class IU_MultiHashIndex(IU_HashIndex): + """ + Class that allows to index more than one key per database record. + + It operates very well on GET/INSERT. It's not optimized for + UPDATE operations (will always readd everything) + """ + + def __init__(self, *args, **kwargs): + super(IU_MultiHashIndex, self).__init__(*args, **kwargs) + + def insert(self, doc_id, key, start, size, status='o'): + if isinstance(key, (list, tuple)): + key = set(key) + elif not isinstance(key, set): + key = set([key]) + ins = super(IU_MultiHashIndex, self).insert + for curr_key in key: + ins(doc_id, curr_key, start, size, status) + return True + + def update(self, doc_id, key, u_start, u_size, u_status='o'): + if isinstance(key, (list, tuple)): + key = set(key) + elif not isinstance(key, set): + key = set([key]) + upd = super(IU_MultiHashIndex, self).update + for curr_key in key: + upd(doc_id, curr_key, u_start, u_size, u_status) + + def delete(self, doc_id, key, start=0, size=0): + if isinstance(key, (list, tuple)): + key = set(key) + elif not isinstance(key, set): + key = set([key]) + delete = super(IU_MultiHashIndex, self).delete + for curr_key in key: + delete(doc_id, curr_key, start, size) + + def get(self, key): + return super(IU_MultiHashIndex, self).get(key) + + def make_key_value(self, data): + raise NotImplementedError() + + +# classes for public use, done in this way because of +# generation static files with indexes (_index directory) + + +class HashIndex(IU_HashIndex): + """ + That class is designed to be used in custom indexes. + """ + pass + + +class UniqueHashIndex(IU_UniqueHashIndex): + """ + That class is designed to be used in custom indexes. It's designed to be **id** index. + """ + pass + + +class MultiHashIndex(IU_MultiHashIndex): + """ + That class is designed to be used in custom indexes. + """ diff --git a/libs/CodernityDB3/index.py b/libs/CodernityDB3/index.py new file mode 100644 index 00000000..104d543b --- /dev/null +++ b/libs/CodernityDB3/index.py @@ -0,0 +1,200 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright 2011-2013 Codernity (http://codernity.com) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +import marshal + +import struct +import shutil + +from CodernityDB3.storage import IU_Storage, DummyStorage + +try: + from CodernityDB3 import __version__ +except ImportError: + from .__init__ import __version__ + + +import io + + +class IndexException(Exception): + pass + + +class IndexNotFoundException(IndexException): + pass + + +class ReindexException(IndexException): + pass + + +class TryReindexException(ReindexException): + pass + + +class ElemNotFound(IndexException): + pass + + +class DocIdNotFound(ElemNotFound): + pass + + +class IndexConflict(IndexException): + pass + + +class IndexPreconditionsException(IndexException): + pass + + +class Index(object): + + __version__ = __version__ + + custom_header = "" # : use it for imports required by your index + + def __init__(self, + db_path, + name): + if isinstance(name, bytes): + name = name.decode() + if isinstance(db_path, bytes): + db_path = db_path.decode() + + self.name = name + self._start_ind = 500 + self.db_path = db_path + + def open_index(self): + if not os.path.isfile(os.path.join(self.db_path, self.name + '_buck')): + raise IndexException("Doesn't exists") + self.buckets = io.open( + os.path.join(self.db_path, self.name + "_buck"), 'r+b', buffering=0) + self._fix_params() + self._open_storage() + + def _close(self): + self.buckets.close() + self.storage.close() + + def close_index(self): + self.flush() + self.fsync() + self._close() + + def create_index(self): + raise NotImplementedError() + + def _fix_params(self): + self.buckets.seek(0) + props = marshal.loads(self.buckets.read(self._start_ind)) + for k, v in props.items(): + self.__dict__[k] = v + self.buckets.seek(0, 2) + + def _save_params(self, in_params={}): + self.buckets.seek(0) + props = marshal.loads(self.buckets.read(self._start_ind)) + props.update(in_params) + self.buckets.seek(0) + data = marshal.dumps(props) + if len(data) > self._start_ind: + raise IndexException("To big props") + self.buckets.write(data) + self.flush() + self.buckets.seek(0, 2) + self.__dict__.update(props) + + def _open_storage(self, *args, **kwargs): + pass + + def _create_storage(self, *args, **kwargs): + pass + + def _destroy_storage(self, *args, **kwargs): + self.storage.destroy() + + def _find_key(self, key): + raise NotImplementedError() + + def update(self, doc_id, key, start, size): + raise NotImplementedError() + + def insert(self, doc_id, key, start, size): + raise NotImplementedError() + + def get(self, key): + raise NotImplementedError() + + def get_many(self, key, start_from=None, limit=0): + raise NotImplementedError() + + def all(self, start_pos): + raise NotImplementedError() + + def delete(self, key, start, size): + raise NotImplementedError() + + def make_key_value(self, data): + raise NotImplementedError() + + def make_key(self, data): + raise NotImplementedError() + + def compact(self, *args, **kwargs): + raise NotImplementedError() + + def destroy(self, *args, **kwargs): + self._close() + bucket_file = os.path.join(self.db_path, self.name + '_buck') + os.unlink(bucket_file) + self._destroy_storage() + self._find_key.clear() + + def flush(self): + try: + self.buckets.flush() + self.storage.flush() + except: + pass + + def fsync(self): + try: + os.fsync(self.buckets.fileno()) + self.storage.fsync() + except: + pass + + def update_with_storage(self, doc_id, key, value): + if value: + start, size = self.storage.insert(value) + else: + start = 1 + size = 0 + return self.update(doc_id, key, start, size) + + def insert_with_storage(self, doc_id, key, value): + if value is not None: + start, size = self.storage.insert(value) + else: + start = 1 + size = 0 + return self.insert(doc_id, key, start, size) diff --git a/libs/CodernityDB3/indexcreator.py b/libs/CodernityDB3/indexcreator.py new file mode 100644 index 00000000..9335340b --- /dev/null +++ b/libs/CodernityDB3/indexcreator.py @@ -0,0 +1,645 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright 2011-2013 Codernity (http://codernity.com) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import re +import tokenize +import token +import uuid + + +class IndexCreatorException(Exception): + def __init__(self, ex, line=None): + self.ex = ex + self.line = line + + def __str__(self): + if self.line: + return repr(self.ex + "(in line: %d)" % self.line) + return repr(self.ex) + + +class IndexCreatorFunctionException(IndexCreatorException): + pass + + +class IndexCreatorValueException(IndexCreatorException): + pass + + +class Parser(object): + def __init__(self): + pass + + def parse(self, data, name=None): + if not name: + self.name = "_" + uuid.uuid4().hex + else: + self.name = name + + self.ind = 0 + self.stage = 0 + self.logic = ['and', 'or', 'in'] + self.logic2 = ['&', '|'] + self.allowed_props = {'TreeBasedIndex': ['type', 'name', 'key_format', 'node_capacity', 'pointer_format', 'meta_format'], + 'HashIndex': ['type', 'name', 'key_format', 'hash_lim', 'entry_line_format'], + 'MultiHashIndex': ['type', 'name', 'key_format', 'hash_lim', 'entry_line_format'], + 'MultiTreeBasedIndex': ['type', 'name', 'key_format', 'node_capacity', 'pointer_format', 'meta_format'] + } + self.funcs = {'md5': (['md5'], ['.digest()']), + 'len': (['len'], []), + 'str': (['str'], []), + 'fix_r': (['self.fix_r'], []), + 'prefix': (['self.prefix'], []), + 'infix': (['self.infix'], []), + 'suffix': (['self.suffix'], []) + } + self.handle_int_imports = {'infix': "from itertools import izip\n"} + + self.funcs_with_body = {'fix_r': + (""" def fix_r(self,s,l): + e = len(s) + if e == l: + return s + elif e > l: + return s[:l] + else: + return s.rjust(l,'_')\n""", False), + 'prefix': + (""" def prefix(self,s,m,l,f): + t = len(s) + if m < 1: + m = 1 + o = set() + if t > l: + s = s[:l] + t = l + while m <= t: + o.add(s.rjust(f,'_')) + s = s[:-1] + t -= 1 + return o\n""", False), + 'suffix': + (""" def suffix(self,s,m,l,f): + t = len(s) + if m < 1: + m = 1 + o = set() + if t > l: + s = s[t-l:] + t = len(s) + while m <= t: + o.add(s.rjust(f,'_')) + s = s[1:] + t -= 1 + return o\n""", False), + 'infix': + (""" def infix(self,s,m,l,f): + t = len(s) + o = set() + for x in xrange(m - 1, l): + t = (s, ) + for y in xrange(0, x): + t += (s[y + 1:],) + o.update(set(''.join(x).rjust(f, '_').lower() for x in izip(*t))) + return o\n""", False)} + self.none = ['None', 'none', 'null'] + self.props_assign = ['=', ':'] + self.all_adj_num_comp = {token.NUMBER: ( + token.NUMBER, token.NAME, '-', '('), + token.NAME: (token.NUMBER, token.NAME, '-', '('), + ')': (token.NUMBER, token.NAME, '-', '(') + } + + self.all_adj_num_op = {token.NUMBER: (token.NUMBER, token.NAME, '('), + token.NAME: (token.NUMBER, token.NAME, '('), + ')': (token.NUMBER, token.NAME, '(') + } + self.allowed_adjacent = { + "<=": self.all_adj_num_comp, + ">=": self.all_adj_num_comp, + ">": self.all_adj_num_comp, + "<": self.all_adj_num_comp, + + "==": {token.NUMBER: (token.NUMBER, token.NAME, '('), + token.NAME: (token.NUMBER, token.NAME, token.STRING, '('), + token.STRING: (token.NAME, token.STRING, '('), + ')': (token.NUMBER, token.NAME, token.STRING, '('), + ']': (token.NUMBER, token.NAME, token.STRING, '(') + }, + + "+": {token.NUMBER: (token.NUMBER, token.NAME, '('), + token.NAME: (token.NUMBER, token.NAME, token.STRING, '('), + token.STRING: (token.NAME, token.STRING, '('), + ')': (token.NUMBER, token.NAME, token.STRING, '('), + ']': (token.NUMBER, token.NAME, token.STRING, '(') + }, + + "-": {token.NUMBER: (token.NUMBER, token.NAME, '('), + token.NAME: (token.NUMBER, token.NAME, '('), + ')': (token.NUMBER, token.NAME, '('), + '<': (token.NUMBER, token.NAME, '('), + '>': (token.NUMBER, token.NAME, '('), + '<=': (token.NUMBER, token.NAME, '('), + '>=': (token.NUMBER, token.NAME, '('), + '==': (token.NUMBER, token.NAME, '('), + ']': (token.NUMBER, token.NAME, '(') + }, + "*": self.all_adj_num_op, + "/": self.all_adj_num_op, + "%": self.all_adj_num_op, + ",": {token.NUMBER: (token.NUMBER, token.NAME, token.STRING, '{', '[', '('), + token.NAME: (token.NUMBER, token.NAME, token.STRING, '(', '{', '['), + token.STRING: (token.NAME, token.STRING, token.NUMBER, '(', '{', '['), + ')': (token.NUMBER, token.NAME, token.STRING, '(', '{', '['), + ']': (token.NUMBER, token.NAME, token.STRING, '(', '{', '['), + '}': (token.NUMBER, token.NAME, token.STRING, '(', '{', '[') + } + } + + def is_num(s): + m = re.search('[^0-9*()+\-\s/]+', s) + return not m + + def is_string(s): + m = re.search('\s*(?P[\'\"]+).*?(?P=a)\s*', s) + return m + data = re.split('make_key_value\:', data) + + if len(data) < 2: + raise IndexCreatorFunctionException( + "Couldn't find a definition of make_key_value function!\n") + + spl1 = re.split('make_key\:', data[0]) + spl2 = re.split('make_key\:', data[1]) + + self.funcs_rev = False + + if len(spl1) > 1: + data = [spl1[0]] + [data[1]] + [spl1[1]] + self.funcs_rev = True + elif len(spl2) > 1: + data = [data[0]] + spl2 + else: + data.append("key") + + if data[1] == re.search('\s*', data[1], re.S | re.M).group(0): + raise IndexCreatorFunctionException("Empty function body ", + len(re.split('\n', data[0])) + (len(re.split('\n', data[2])) if self.funcs_rev else 1) - 1) + if data[2] == re.search('\s*', data[2], re.S | re.M).group(0): + raise IndexCreatorFunctionException("Empty function body ", + len(re.split('\n', data[0])) + (1 if self.funcs_rev else len(re.split('\n', data[1]))) - 1) + if data[0] == re.search('\s*', data[0], re.S | re.M).group(0): + raise IndexCreatorValueException("You didn't set any properity or you set them not at the begining of the code\n") + + data = [re.split( + '\n', data[0]), re.split('\n', data[1]), re.split('\n', data[2])] + self.cnt_lines = (len(data[0]), len(data[1]), len(data[2])) + ind = 0 + self.predata = data + self.data = [[], [], []] + for i, v in enumerate(self.predata[0]): + for k, w in enumerate(self.predata[0][i]): + if self.predata[0][i][k] in self.props_assign: + if not is_num(self.predata[0][i][k + 1:]) and self.predata[0][i].strip()[:4] != 'type' and self.predata[0][i].strip()[:4] != 'name': + s = self.predata[0][i][k + 1:] + self.predata[0][i] = self.predata[0][i][:k + 1] + + m = re.search('\s+', s.strip()) + if not is_string(s) and not m: + s = "'" + s.strip() + "'" + self.predata[0][i] += s + break + + for n, i in enumerate(self.predata): + for k in i: + k = k.strip() + if k: + self.data[ind].append(k) + self.check_enclosures(k, n) + ind += 1 + + return self.parse_ex() + + def readline(self, stage): + def foo(): + if len(self.data[stage]) <= self.ind: + self.ind = 0 + return "" + else: + self.ind += 1 + return self.data[stage][self.ind - 1] + return foo + + def add(self, l, i): + def add_aux(*args): + # print args,self.ind + if len(l[i]) < self.ind: + l[i].append([]) + l[i][self.ind - 1].append(args) + return add_aux + + def parse_ex(self): + self.index_name = "" + self.index_type = "" + self.curLine = -1 + self.con = -1 + self.brackets = -1 + self.curFunc = None + self.colons = 0 + self.line_cons = ([], [], []) + self.pre_tokens = ([], [], []) + self.known_dicts_in_mkv = [] + self.prop_name = True + self.prop_assign = False + self.is_one_arg_enough = False + self.funcs_stack = [] + self.last_line = [-1, -1, -1] + self.props_set = [] + self.custom_header = set() + + self.tokens = [] + self.tokens_head = ['# %s\n' % self.name, 'class %s(' % self.name, '):\n', ' def __init__(self, *args, **kwargs): '] + + for i in range(3): + tokenize.tokenize(self.readline(i), self.add(self.pre_tokens, i)) + # tokenize treats some keyword not in the right way, thats why we + # have to change some of them + for nk, k in enumerate(self.pre_tokens[i]): + for na, a in enumerate(k): + if a[0] == token.NAME and a[1] in self.logic: + self.pre_tokens[i][nk][ + na] = (token.OP, a[1], a[2], a[3], a[4]) + + for i in self.pre_tokens[1]: + self.line_cons[1].append(self.check_colons(i, 1)) + self.check_adjacents(i, 1) + if self.check_for_2nd_arg(i) == -1 and not self.is_one_arg_enough: + raise IndexCreatorValueException("No 2nd value to return (did u forget about ',None'?", self.cnt_line_nr(i[0][4], 1)) + self.is_one_arg_enough = False + + for i in self.pre_tokens[2]: + self.line_cons[2].append(self.check_colons(i, 2)) + self.check_adjacents(i, 2) + + for i in self.pre_tokens[0]: + self.handle_prop_line(i) + + self.cur_brackets = 0 + self.tokens += ['\n super(%s, self).__init__(*args, **kwargs)\n def make_key_value(self, data): ' % self.name] + + for i in self.pre_tokens[1]: + for k in i: + self.handle_make_value(*k) + + self.curLine = -1 + self.con = -1 + self.cur_brackets = 0 + self.tokens += ['\n def make_key(self, key):'] + + for i in self.pre_tokens[2]: + for k in i: + self.handle_make_key(*k) + + if self.index_type == "": + raise IndexCreatorValueException("Missing index type definition\n") + if self.index_name == "": + raise IndexCreatorValueException("Missing index name\n") + + self.tokens_head[0] = "# " + self.index_name + "\n" + \ + self.tokens_head[0] + + for i in self.funcs_with_body: + if self.funcs_with_body[i][1]: + self.tokens_head.insert(4, self.funcs_with_body[i][0]) + + if None in self.custom_header: + self.custom_header.remove(None) + if self.custom_header: + s = ' custom_header = """' + for i in self.custom_header: + s += i + s += '"""\n' + self.tokens_head.insert(4, s) + + if self.index_type in self.allowed_props: + for i in self.props_set: + if i not in self.allowed_props[self.index_type]: + raise IndexCreatorValueException("Properity %s is not allowed for index type: %s" % (i, self.index_type)) + + # print "".join(self.tokens_head) + # print "----------" + # print (" ".join(self.tokens)) + return "".join(self.custom_header), "".join(self.tokens_head) + (" ".join(self.tokens)) + + # has to be run BEFORE tokenize + def check_enclosures(self, d, st): + encs = [] + contr = {'(': ')', '{': '}', '[': ']', "'": "'", '"': '"'} + ends = [')', '}', ']', "'", '"'] + for i in d: + if len(encs) > 0 and encs[-1] in ['"', "'"]: + if encs[-1] == i: + del encs[-1] + elif i in contr: + encs += [i] + elif i in ends: + if len(encs) < 1 or contr[encs[-1]] != i: + raise IndexCreatorValueException("Missing opening enclosure for \'%s\'" % i, self.cnt_line_nr(d, st)) + del encs[-1] + + if len(encs) > 0: + raise IndexCreatorValueException("Missing closing enclosure for \'%s\'" % encs[0], self.cnt_line_nr(d, st)) + + def check_adjacents(self, d, st): + def std_check(d, n): + if n == 0: + prev = -1 + else: + prev = d[n - 1][1] if d[n - 1][0] == token.OP else d[n - 1][0] + + cur = d[n][1] if d[n][0] == token.OP else d[n][0] + + # there always is an endmarker at the end, but this is a precaution + if n + 2 > len(d): + nex = -1 + else: + nex = d[n + 1][1] if d[n + 1][0] == token.OP else d[n + 1][0] + + if prev not in self.allowed_adjacent[cur]: + raise IndexCreatorValueException("Wrong left value of the %s" % cur, self.cnt_line_nr(line, st)) + + # there is an assumption that whole data always ends with 0 marker, the idea prolly needs a rewritting to allow more whitespaces + # between tokens, so it will be handled anyway + elif nex not in self.allowed_adjacent[cur][prev]: + raise IndexCreatorValueException("Wrong right value of the %s" % cur, self.cnt_line_nr(line, st)) + + for n, (t, i, _, _, line) in enumerate(d): + if t == token.NAME or t == token.STRING: + if n + 1 < len(d) and d[n + 1][0] in [token.NAME, token.STRING]: + raise IndexCreatorValueException("Did you forget about an operator in between?", self.cnt_line_nr(line, st)) + elif i in self.allowed_adjacent: + std_check(d, n) + + def check_colons(self, d, st): + cnt = 0 + br = 0 + + def check_ret_args_nr(a, s): + c_b_cnt = 0 + s_b_cnt = 0 + n_b_cnt = 0 + comas_cnt = 0 + for _, i, _, _, line in a: + + if c_b_cnt == n_b_cnt == s_b_cnt == 0: + if i == ',': + comas_cnt += 1 + if (s == 1 and comas_cnt > 1) or (s == 2 and comas_cnt > 0): + raise IndexCreatorFunctionException("Too much arguments to return", self.cnt_line_nr(line, st)) + if s == 0 and comas_cnt > 0: + raise IndexCreatorValueException("A coma here doesn't make any sense", self.cnt_line_nr(line, st)) + + elif i == ':': + if s == 0: + raise IndexCreatorValueException("A colon here doesn't make any sense", self.cnt_line_nr(line, st)) + raise IndexCreatorFunctionException("Two colons don't make any sense", self.cnt_line_nr(line, st)) + + if i == '{': + c_b_cnt += 1 + elif i == '}': + c_b_cnt -= 1 + elif i == '(': + n_b_cnt += 1 + elif i == ')': + n_b_cnt -= 1 + elif i == '[': + s_b_cnt += 1 + elif i == ']': + s_b_cnt -= 1 + + def check_if_empty(a): + for i in a: + if i not in [token.NEWLINE, token.INDENT, token.ENDMARKER]: + return False + return True + if st == 0: + check_ret_args_nr(d, st) + return + + for n, i in enumerate(d): + if i[1] == ':': + if br == 0: + if len(d) < n or check_if_empty(d[n + 1:]): + raise IndexCreatorValueException( + "Empty return value", self.cnt_line_nr(i[4], st)) + elif len(d) >= n: + check_ret_args_nr(d[n + 1:], st) + return cnt + else: + cnt += 1 + elif i[1] == '{': + br += 1 + elif i[1] == '}': + br -= 1 + check_ret_args_nr(d, st) + return -1 + + def check_for_2nd_arg(self, d): + c_b_cnt = 0 # curly brackets counter '{}' + s_b_cnt = 0 # square brackets counter '[]' + n_b_cnt = 0 # normal brackets counter '()' + + def check_2nd_arg(d, ind): + d = d[ind[0]:] + for t, i, (n, r), _, line in d: + if i == '{' or i is None: + return 0 + elif t == token.NAME: + self.known_dicts_in_mkv.append((i, (n, r))) + return 0 + elif t == token.STRING or t == token.NUMBER: + raise IndexCreatorValueException("Second return value of make_key_value function has to be a dictionary!", self.cnt_line_nr(line, 1)) + + for ind in enumerate(d): + t, i, _, _, _ = ind[1] + if s_b_cnt == n_b_cnt == c_b_cnt == 0: + if i == ',': + return check_2nd_arg(d, ind) + elif (t == token.NAME and i not in self.funcs) or i == '{': + self.is_one_arg_enough = True + + if i == '{': + c_b_cnt += 1 + self.is_one_arg_enough = True + elif i == '}': + c_b_cnt -= 1 + elif i == '(': + n_b_cnt += 1 + elif i == ')': + n_b_cnt -= 1 + elif i == '[': + s_b_cnt += 1 + elif i == ']': + s_b_cnt -= 1 + return -1 + + def cnt_line_nr(self, l, stage): + nr = -1 + for n, i in enumerate(self.predata[stage]): + # print i,"|||",i.strip(),"|||",l + if l == i.strip(): + nr = n + if nr == -1: + return -1 + + if stage == 0: + return nr + 1 + elif stage == 1: + return nr + self.cnt_lines[0] + (self.cnt_lines[2] - 1 if self.funcs_rev else 0) + elif stage == 2: + return nr + self.cnt_lines[0] + (self.cnt_lines[1] - 1 if not self.funcs_rev else 0) + + return -1 + + def handle_prop_line(self, d): + d_len = len(d) + if d[d_len - 1][0] == token.ENDMARKER: + d_len -= 1 + + if d_len < 3: + raise IndexCreatorValueException("Can't handle properity assingment ", self.cnt_line_nr(d[0][4], 0)) + + if not d[1][1] in self.props_assign: + raise IndexCreatorValueException( + "Did you forget : or =?", self.cnt_line_nr(d[0][4], 0)) + + if d[0][0] == token.NAME or d[0][0] == token.STRING: + if d[0][1] in self.props_set: + raise IndexCreatorValueException("Properity %s is set more than once" % d[0][1], self.cnt_line_nr(d[0][4], 0)) + self.props_set += [d[0][1]] + if d[0][1] == "type" or d[0][1] == "name": + t, tk, _, _, line = d[2] + + if d_len > 3: + raise IndexCreatorValueException( + "Wrong value to assign", self.cnt_line_nr(line, 0)) + + if t == token.STRING: + m = re.search('\s*(?P[\'\"]+)(.*?)(?P=a)\s*', tk) + if m: + tk = m.groups()[1] + elif t != token.NAME: + raise IndexCreatorValueException( + "Wrong value to assign", self.cnt_line_nr(line, 0)) + + if d[0][1] == "type": + if d[2][1] == "TreeBasedIndex": + self.custom_header.add("from CodernityDB3.tree_index import TreeBasedIndex\n") + elif d[2][1] == "MultiTreeBasedIndex": + self.custom_header.add("from CodernityDB3.tree_index import MultiTreeBasedIndex\n") + elif d[2][1] == "MultiHashIndex": + self.custom_header.add("from CodernityDB3.hash_index import MultiHashIndex\n") + self.tokens_head.insert(2, tk) + self.index_type = tk + else: + self.index_name = tk + return + else: + self.tokens += ['\n kwargs["' + d[0][1] + '"]'] + else: + raise IndexCreatorValueException("Can't handle properity assingment ", self.cnt_line_nr(d[0][4], 0)) + + self.tokens += ['='] + + self.check_adjacents(d[2:], 0) + self.check_colons(d[2:], 0) + + for i in d[2:]: + self.tokens += [i[1]] + + def generate_func(self, t, tk, pos_start, pos_end, line, hdata, stage): + if self.last_line[stage] != -1 and pos_start[0] > self.last_line[stage] and line != '': + raise IndexCreatorFunctionException("This line will never be executed!", self.cnt_line_nr(line, stage)) + if t == 0: + return + + if pos_start[1] == 0: + if self.line_cons[stage][pos_start[0] - 1] == -1: + self.tokens += ['\n return'] + self.last_line[stage] = pos_start[0] + else: + self.tokens += ['\n if'] + elif tk == ':' and self.line_cons[stage][pos_start[0] - 1] > -1: + if self.line_cons[stage][pos_start[0] - 1] == 0: + self.tokens += [':\n return'] + return + self.line_cons[stage][pos_start[0] - 1] -= 1 + + if tk in self.logic2: + # print tk + if line[pos_start[1] - 1] != tk and line[pos_start[1] + 1] != tk: + self.tokens += [tk] + if line[pos_start[1] - 1] != tk and line[pos_start[1] + 1] == tk: + if tk == '&': + self.tokens += ['and'] + else: + self.tokens += ['or'] + return + + if self.brackets != 0: + def search_through_known_dicts(a): + for i, (n, r) in self.known_dicts_in_mkv: + if i == tk and r > pos_start[1] and n == pos_start[0] and hdata == 'data': + return True + return False + + if t == token.NAME and len(self.funcs_stack) > 0 and self.funcs_stack[-1][0] == 'md5' and search_through_known_dicts(tk): + raise IndexCreatorValueException("Second value returned by make_key_value for sure isn't a dictionary ", self.cnt_line_nr(line, 1)) + + if tk == ')': + self.cur_brackets -= 1 + if len(self.funcs_stack) > 0 and self.cur_brackets == self.funcs_stack[-1][1]: + self.tokens += [tk] + self.tokens += self.funcs[self.funcs_stack[-1][0]][1] + del self.funcs_stack[-1] + return + if tk == '(': + self.cur_brackets += 1 + + if tk in self.none: + self.tokens += ['None'] + return + + if t == token.NAME and tk not in self.logic and tk != hdata: + if tk not in self.funcs: + self.tokens += [hdata + '["' + tk + '"]'] + else: + self.tokens += self.funcs[tk][0] + if tk in self.funcs_with_body: + self.funcs_with_body[tk] = ( + self.funcs_with_body[tk][0], True) + self.custom_header.add(self.handle_int_imports.get(tk)) + self.funcs_stack += [(tk, self.cur_brackets)] + else: + self.tokens += [tk] + + def handle_make_value(self, t, tk, pos_start, pos_end, line): + self.generate_func(t, tk, pos_start, pos_end, line, 'data', 1) + + def handle_make_key(self, t, tk, pos_start, pos_end, line): + self.generate_func(t, tk, pos_start, pos_end, line, 'key', 2) diff --git a/libs/CodernityDB3/lfu_cache.py b/libs/CodernityDB3/lfu_cache.py new file mode 100644 index 00000000..5b784808 --- /dev/null +++ b/libs/CodernityDB3/lfu_cache.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright 2011-2013 Codernity (http://codernity.com) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import functools +from heapq import nsmallest +from operator import itemgetter +from collections import defaultdict + +try: + from collections import Counter +except ImportError: + class Counter(dict): + 'Mapping where default values are zero' + def __missing__(self, key): + return 0 + + +def cache1lvl(maxsize=100): + """ + modified version of http://code.activestate.com/recipes/498245/ + """ + def decorating_function(user_function): + cache = {} + use_count = Counter() + + @functools.wraps(user_function) + def wrapper(key, *args, **kwargs): + try: + result = cache[key] + except KeyError: + if len(cache) == maxsize: + for k, _ in nsmallest(maxsize // 10 or 1, + iter(use_count.items()), + key=itemgetter(1)): + del cache[k], use_count[k] + cache[key] = user_function(key, *args, **kwargs) + result = cache[key] + # result = user_function(obj, key, *args, **kwargs) + finally: + use_count[key] += 1 + return result + + def clear(): + cache.clear() + use_count.clear() + + def delete(key): + try: + del cache[key] + del use_count[key] + except KeyError: + return False + else: + return True + + wrapper.clear = clear + wrapper.cache = cache + wrapper.delete = delete + return wrapper + return decorating_function + + +def twolvl_iterator(dict): + for k, v in dict.items(): + for kk, vv in v.items(): + yield k, kk, vv + + +def cache2lvl(maxsize=100): + """ + modified version of http://code.activestate.com/recipes/498245/ + """ + def decorating_function(user_function): + cache = {} + use_count = defaultdict(Counter) + + @functools.wraps(user_function) + def wrapper(*args, **kwargs): +# return user_function(*args, **kwargs) + try: + result = cache[args[0]][args[1]] + except KeyError: + if wrapper.cache_size == maxsize: + to_delete = maxsize // 10 or 1 + for k1, k2, v in nsmallest(to_delete, + twolvl_iterator(use_count), + key=itemgetter(2)): + del cache[k1][k2], use_count[k1][k2] + if not cache[k1]: + del cache[k1] + del use_count[k1] + wrapper.cache_size -= to_delete + result = user_function(*args, **kwargs) + try: + cache[args[0]][args[1]] = result + except KeyError: + cache[args[0]] = {args[1]: result} + wrapper.cache_size += 1 + finally: + use_count[args[0]][args[1]] += 1 + return result + + def clear(): + cache.clear() + use_count.clear() + + def delete(key, inner_key=None): + if inner_key is not None: + try: + del cache[key][inner_key] + del use_count[key][inner_key] + if not cache[key]: + del cache[key] + del use_count[key] + wrapper.cache_size -= 1 + except KeyError: + return False + else: + return True + else: + try: + wrapper.cache_size -= len(cache[key]) + del cache[key] + del use_count[key] + except KeyError: + return False + else: + return True + + wrapper.clear = clear + wrapper.cache = cache + wrapper.delete = delete + wrapper.cache_size = 0 + return wrapper + return decorating_function diff --git a/libs/CodernityDB3/lfu_cache_with_lock.py b/libs/CodernityDB3/lfu_cache_with_lock.py new file mode 100644 index 00000000..15150071 --- /dev/null +++ b/libs/CodernityDB3/lfu_cache_with_lock.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright 2011-2013 Codernity (http://codernity.com) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import functools +from heapq import nsmallest +from operator import itemgetter +from collections import defaultdict + + +try: + from collections import Counter +except ImportError: + class Counter(dict): + 'Mapping where default values are zero' + def __missing__(self, key): + return 0 + + +def twolvl_iterator(dict): + for k, v in dict.items(): + for kk, vv in v.items(): + yield k, kk, vv + + +def create_cache1lvl(lock_obj): + def cache1lvl(maxsize=100): + """ + modified version of http://code.activestate.com/recipes/498245/ + """ + def decorating_function(user_function): + cache = {} + use_count = Counter() + lock = lock_obj() + + @functools.wraps(user_function) + def wrapper(key, *args, **kwargs): + try: + result = cache[key] + except KeyError: + with lock: + if len(cache) == maxsize: + for k, _ in nsmallest(maxsize // 10 or 1, + iter(use_count.items()), + key=itemgetter(1)): + del cache[k], use_count[k] + cache[key] = user_function(key, *args, **kwargs) + result = cache[key] + use_count[key] += 1 + else: + with lock: + use_count[key] += 1 + return result + + def clear(): + cache.clear() + use_count.clear() + + def delete(key): + try: + del cache[key] + del use_count[key] + return True + except KeyError: + return False + + wrapper.clear = clear + wrapper.cache = cache + wrapper.delete = delete + return wrapper + return decorating_function + return cache1lvl + + +def create_cache2lvl(lock_obj): + def cache2lvl(maxsize=100): + """ + modified version of http://code.activestate.com/recipes/498245/ + """ + def decorating_function(user_function): + cache = {} + use_count = defaultdict(Counter) + lock = lock_obj() + + @functools.wraps(user_function) + def wrapper(*args, **kwargs): + try: + result = cache[args[0]][args[1]] + except KeyError: + with lock: + if wrapper.cache_size == maxsize: + to_delete = maxsize / 10 or 1 + for k1, k2, v in nsmallest(to_delete, + twolvl_iterator( + use_count), + key=itemgetter(2)): + del cache[k1][k2], use_count[k1][k2] + if not cache[k1]: + del cache[k1] + del use_count[k1] + wrapper.cache_size -= to_delete + result = user_function(*args, **kwargs) + try: + cache[args[0]][args[1]] = result + except KeyError: + cache[args[0]] = {args[1]: result} + use_count[args[0]][args[1]] += 1 + wrapper.cache_size += 1 + else: + use_count[args[0]][args[1]] += 1 + return result + + def clear(): + cache.clear() + use_count.clear() + + def delete(key, *args): + if args: + try: + del cache[key][args[0]] + del use_count[key][args[0]] + if not cache[key]: + del cache[key] + del use_count[key] + wrapper.cache_size -= 1 + return True + except KeyError: + return False + else: + try: + wrapper.cache_size -= len(cache[key]) + del cache[key] + del use_count[key] + return True + except KeyError: + return False + + wrapper.clear = clear + wrapper.cache = cache + wrapper.delete = delete + wrapper.cache_size = 0 + return wrapper + return decorating_function + return cache2lvl diff --git a/libs/CodernityDB3/migrate.py b/libs/CodernityDB3/migrate.py new file mode 100644 index 00000000..393e82c7 --- /dev/null +++ b/libs/CodernityDB3/migrate.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright 2011-2013 Codernity (http://codernity.com) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from CodernityDB3.database import Database +import shutil +import os + + +def migrate(source, destination): + """ + Very basic for now + """ + dbs = Database(source) + dbt = Database(destination) + dbs.open() + dbt.create() + dbt.close() + for curr in os.listdir(os.path.join(dbs.path, '_indexes')): + if curr != '00id.py': + shutil.copyfile(os.path.join(dbs.path, '_indexes', curr), + os.path.join(dbt.path, '_indexes', curr)) + dbt.open() + for c in dbs.all('id'): + del c['_rev'] + dbt.insert(c) + return True + + +if __name__ == '__main__': + import sys + migrate(sys.argv[1], sys.argv[2]) diff --git a/libs/CodernityDB3/misc.py b/libs/CodernityDB3/misc.py new file mode 100644 index 00000000..54c94812 --- /dev/null +++ b/libs/CodernityDB3/misc.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright 2011-2013 Codernity (http://codernity.com) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from random import getrandbits, randrange +import uuid + + +class NONE: + """ + It's inteded to be None but different, + for internal use only! + """ + pass + + +def random_hex_32(): + return uuid.UUID(int=getrandbits(128), version=4).hex + + +def random_hex_4(*args, **kwargs): + return '%04x' % randrange(256 ** 2) diff --git a/libs/CodernityDB3/patch.py b/libs/CodernityDB3/patch.py new file mode 100644 index 00000000..d03c4456 --- /dev/null +++ b/libs/CodernityDB3/patch.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright 2011-2013 Codernity (http://codernity.com) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from CodernityDB3.misc import NONE + + +def __patch(obj, name, new): + n = NONE() + orig = getattr(obj, name, n) + if orig is not n: + if orig == new: + raise Exception("Shouldn't happen, new and orig are the same") + setattr(obj, name, new) + return + + +def patch_cache_lfu(lock_obj): + """ + Patnches cache mechanizm to be thread safe (gevent ones also) + + .. note:: + + It's internal CodernityDB mechanizm, it will be called when needed + + """ + from . import lfu_cache + from . import lfu_cache_with_lock + lfu_lock1lvl = lfu_cache_with_lock.create_cache1lvl(lock_obj) + lfu_lock2lvl = lfu_cache_with_lock.create_cache2lvl(lock_obj) + __patch(lfu_cache, 'cache1lvl', lfu_lock1lvl) + __patch(lfu_cache, 'cache2lvl', lfu_lock2lvl) + + +def patch_cache_rr(lock_obj): + """ + Patches cache mechanizm to be thread safe (gevent ones also) + + .. note:: + + It's internal CodernityDB mechanizm, it will be called when needed + + """ + from . import rr_cache + from . import rr_cache_with_lock + rr_lock1lvl = rr_cache_with_lock.create_cache1lvl(lock_obj) + rr_lock2lvl = rr_cache_with_lock.create_cache2lvl(lock_obj) + __patch(rr_cache, 'cache1lvl', rr_lock1lvl) + __patch(rr_cache, 'cache2lvl', rr_lock2lvl) + + +def patch_flush_fsync(db_obj): + """ + Will always execute index.fsync after index.flush. + + .. note:: + + It's for advanced users, use when you understand difference between `flush` and `fsync`, and when you definitely need that. + + It's important to call it **AFTER** database has all indexes etc (after db.create or db.open) + + Example usage:: + + ... + db = Database('/tmp/patch_demo') + db.create() + patch_flush_fsync(db) + ... + + """ + + def always_fsync(ind_obj): + def _inner(): + ind_obj.orig_flush() + ind_obj.fsync() + return _inner + + for index in db_obj.indexes: + setattr(index, 'orig_flush', index.flush) + setattr(index, 'flush', always_fsync(index)) + + setattr(db_obj, 'orig_flush', db_obj.flush) + setattr(db_obj, 'flush', always_fsync(db_obj)) + + return diff --git a/libs/CodernityDB3/rr_cache.py b/libs/CodernityDB3/rr_cache.py new file mode 100644 index 00000000..fc4d1886 --- /dev/null +++ b/libs/CodernityDB3/rr_cache.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright 2011-2013 Codernity (http://codernity.com) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import functools +from random import choice + + +def cache1lvl(maxsize=100): + def decorating_function(user_function): + cache1lvl = {} + + @functools.wraps(user_function) + def wrapper(key, *args, **kwargs): + if isinstance(key, bytes): + key = key.decode() + # print("cachedddd", key) ## TODO + try: + #result = cache1lvl[key] + result = cache1lvl[key] + except KeyError: + if len(cache1lvl) == maxsize: + for i in range(maxsize // 10 or 1): + del cache1lvl[choice(list(cache1lvl.keys()))] + ## print("#" * 10, key) # TODO + ## print(user_function) # TODO + ## print("cache1lvl", key, user_function) # TODO + ## print(cache1lvl) # TODO + cache1lvl[key] = user_function(key, *args, **kwargs) + ## print(cache1lvl) # TODO + result = cache1lvl[key] + ## print("result caching", result) # TODO +# result = user_function(obj, key, *args, **kwargs) + if isinstance(result, bytes): + result = key.decode() + ## print("r" * 20, result) # TODO + return result + + def clear(): + cache1lvl.clear() + + def delete(key): + if isinstance(key, bytes): + key = key.decode() + try: + del cache1lvl[key] + return True + except KeyError: + return False + + wrapper.clear = clear + wrapper.cache = cache1lvl + wrapper.delete = delete + return wrapper + return decorating_function + + +def cache2lvl(maxsize=100): + def decorating_function(user_function): + cache = {} + + @functools.wraps(user_function) + def wrapper(*args, **kwargs): +# return user_function(*args, **kwargs) + try: + result = cache[args[0]][args[1]] + except KeyError: +# print wrapper.cache_size + if wrapper.cache_size == maxsize: + to_delete = maxsize // 10 or 1 + for i in range(to_delete): + key1 = choice(list(cache.keys())) + key2 = choice(list(cache[key1].keys())) + del cache[key1][key2] + if not cache[key1]: + del cache[key1] + wrapper.cache_size -= to_delete +# print wrapper.cache_size + result = user_function(*args, **kwargs) + try: + cache[args[0]][args[1]] = result + except KeyError: + cache[args[0]] = {args[1]: result} + wrapper.cache_size += 1 + return result + + def clear(): + cache.clear() + wrapper.cache_size = 0 + + def delete(key, inner_key=None): + if inner_key: + try: + del cache[key][inner_key] + if not cache[key]: + del cache[key] + wrapper.cache_size -= 1 + return True + except KeyError: + return False + else: + try: + wrapper.cache_size -= len(cache[key]) + del cache[key] + return True + except KeyError: + return False + + wrapper.clear = clear + wrapper.cache = cache + wrapper.delete = delete + wrapper.cache_size = 0 + return wrapper + return decorating_function diff --git a/libs/CodernityDB3/rr_cache_with_lock.py b/libs/CodernityDB3/rr_cache_with_lock.py new file mode 100644 index 00000000..3151daa9 --- /dev/null +++ b/libs/CodernityDB3/rr_cache_with_lock.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright 2011-2013 Codernity (http://codernity.com) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import functools +from random import choice + + +def create_cache1lvl(lock_obj): + def cache1lvl(maxsize=100): + def decorating_function(user_function): + cache = {} + lock = lock_obj() + + @functools.wraps(user_function) + def wrapper(key, *args, **kwargs): + try: + result = cache[key] + except KeyError: + with lock: + if len(cache) == maxsize: + for i in range(maxsize // 10 or 1): + del cache[choice(list(cache.keys()))] + cache[key] = user_function(key, *args, **kwargs) + result = cache[key] + return result + + def clear(): + cache.clear() + + def delete(key): + try: + del cache[key] + return True + except KeyError: + return False + + wrapper.clear = clear + wrapper.cache = cache + wrapper.delete = delete + return wrapper + return decorating_function + return cache1lvl + + +def create_cache2lvl(lock_obj): + def cache2lvl(maxsize=100): + def decorating_function(user_function): + cache = {} + lock = lock_obj() + + @functools.wraps(user_function) + def wrapper(*args, **kwargs): + try: + result = cache[args[0]][args[1]] + except KeyError: + with lock: + if wrapper.cache_size == maxsize: + to_delete = maxsize // 10 or 1 + for i in range(to_delete): + key1 = choice(list(cache.keys())) + key2 = choice(list(cache[key1].keys())) + del cache[key1][key2] + if not cache[key1]: + del cache[key1] + wrapper.cache_size -= to_delete + result = user_function(*args, **kwargs) + try: + cache[args[0]][args[1]] = result + except KeyError: + cache[args[0]] = {args[1]: result} + wrapper.cache_size += 1 + return result + + def clear(): + cache.clear() + wrapper.cache_size = 0 + + def delete(key, *args): + if args: + try: + del cache[key][args[0]] + if not cache[key]: + del cache[key] + wrapper.cache_size -= 1 + return True + except KeyError: + return False + else: + try: + wrapper.cache_size -= len(cache[key]) + del cache[key] + return True + except KeyError: + return False + + wrapper.clear = clear + wrapper.cache = cache + wrapper.delete = delete + wrapper.cache_size = 0 + return wrapper + return decorating_function + return cache2lvl diff --git a/libs/CodernityDB3/sharded_hash.py b/libs/CodernityDB3/sharded_hash.py new file mode 100644 index 00000000..ea0b5250 --- /dev/null +++ b/libs/CodernityDB3/sharded_hash.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright 2011-2013 Codernity (http://codernity.com) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from CodernityDB3.hash_index import UniqueHashIndex, HashIndex +from CodernityDB3.sharded_index import ShardedIndex +from CodernityDB3.index import IndexPreconditionsException + +from random import getrandbits +import uuid + + +class IU_ShardedUniqueHashIndex(ShardedIndex): + + custom_header = """import uuid +from random import getrandbits +from CodernityDB3.sharded_index import ShardedIndex +""" + + def __init__(self, db_path, name, *args, **kwargs): + if kwargs.get('sh_nums', 0) > 255: + raise IndexPreconditionsException("Too many shards") + kwargs['ind_class'] = UniqueHashIndex + super(IU_ShardedUniqueHashIndex, self).__init__(db_path, + name, *args, **kwargs) + self.patchers.append(self.wrap_insert_id_index) + + @staticmethod + def wrap_insert_id_index(db_obj, clean=False): + def _insert_id_index(_rev, data): + """ + Performs insert on **id** index. + """ + _id, value = db_obj.id_ind.make_key_value(data) # may be improved + trg_shard = _id[:2] + storage = db_obj.id_ind.shards_r[trg_shard].storage + start, size = storage.insert(value) + db_obj.id_ind.insert(_id, _rev, start, size) + return _id + if not clean: + if hasattr(db_obj, '_insert_id_index_orig'): + raise IndexPreconditionsException( + "Already patched, something went wrong") + setattr(db_obj, "_insert_id_index_orig", db_obj._insert_id_index) + setattr(db_obj, "_insert_id_index", _insert_id_index) + else: + setattr(db_obj, "_insert_id_index", db_obj._insert_id_index_orig) + delattr(db_obj, "_insert_id_index_orig") + + def create_key(self): + h = uuid.UUID(int=getrandbits(128), version=4).hex + trg = self.last_used + 1 + if trg >= self.sh_nums: + trg = 0 + self.last_used = trg + h = '%02x%30s' % (trg, h[2:]) + return h + + def delete(self, key, *args, **kwargs): + trg_shard = key[:2] + op = self.shards_r[trg_shard] + return op.delete(key, *args, **kwargs) + + def update(self, key, *args, **kwargs): + trg_shard = key[:2] + self.last_used = int(trg_shard, 16) + op = self.shards_r[trg_shard] + return op.update(key, *args, **kwargs) + + def insert(self, key, *args, **kwargs): + trg_shard = key[:2] # in most cases it's in create_key BUT not always + self.last_used = int(key[:2], 16) + op = self.shards_r[trg_shard] + return op.insert(key, *args, **kwargs) + + def get(self, key, *args, **kwargs): + trg_shard = key[:2] + self.last_used = int(trg_shard, 16) + op = self.shards_r[trg_shard] + return op.get(key, *args, **kwargs) + + +class ShardedUniqueHashIndex(IU_ShardedUniqueHashIndex): + + # allow unique hash to be used directly + custom_header = 'from CodernityDB3.sharded_hash import IU_ShardedUniqueHashIndex' + + pass + + +class IU_ShardedHashIndex(ShardedIndex): + + custom_header = """from CodernityDB3.sharded_index import ShardedIndex""" + + def __init__(self, db_path, name, *args, **kwargs): + kwargs['ind_class'] = HashIndex + super(IU_ShardedHashIndex, self).__init__(db_path, name, * + args, **kwargs) + + def calculate_shard(self, key): + """ + Must be implemented. It has to return shard to be used by key + + :param key: key + :returns: target shard + :rtype: int + """ + raise NotImplementedError() + + def delete(self, doc_id, key, *args, **kwargs): + trg_shard = self.calculate_shard(key) + op = self.shards_r[trg_shard] + return op.delete(doc_id, key, *args, **kwargs) + + def insert(self, doc_id, key, *args, **kwargs): + trg_shard = self.calculate_shard(key) + op = self.shards_r[trg_shard] + return op.insert(doc_id, key, *args, **kwargs) + + def update(self, doc_id, key, *args, **kwargs): + trg_shard = self.calculate_shard(key) + op = self.shards_r[trg_shard] + return op.insert(doc_id, key, *args, **kwargs) + + def get(self, key, *args, **kwargs): + trg_shard = self.calculate_shard(key) + op = self.shards_r[trg_shard] + return op.get(key, *args, **kwargs) + + +class ShardedHashIndex(IU_ShardedHashIndex): + pass diff --git a/libs/CodernityDB3/sharded_index.py b/libs/CodernityDB3/sharded_index.py new file mode 100644 index 00000000..d2fb2234 --- /dev/null +++ b/libs/CodernityDB3/sharded_index.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright 2011-2013 Codernity (http://codernity.com) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from CodernityDB3.index import Index +# from CodernityDB3.env import cdb_environment +# import warnings + + +class ShardedIndex(Index): + + def __init__(self, db_path, name, *args, **kwargs): + """ + There are 3 additional parameters. You have to hardcode them in your custom class. **NEVER** use directly + + :param int sh_nums: how many shards should be + :param class ind_class: Index class to use (HashIndex or your custom one) + :param bool use_make_keys: if True, `make_key`, and `make_key_value` will be overriden with those from first shard + + The rest parameters are passed straight to `ind_class` shards. + + """ + super(ShardedIndex, self).__init__(db_path, name) + try: + self.sh_nums = kwargs.pop('sh_nums') + except KeyError: + self.sh_nums = 5 + try: + ind_class = kwargs.pop('ind_class') + except KeyError: + raise Exception("ind_class must be given") + else: + # if not isinstance(ind_class, basestring): + # ind_class = ind_class.__name__ + self.ind_class = ind_class + if 'use_make_keys' in kwargs: + self.use_make_keys = kwargs.pop('use_make_keys') + else: + self.use_make_keys = False + self._set_shard_datas(*args, **kwargs) + self.patchers = [] # database object patchers + + def _set_shard_datas(self, *args, **kwargs): + self.shards = {} + self.shards_r = {} +# ind_class = globals()[self.ind_class] + ind_class = self.ind_class + i = 0 + for sh_name in [self.name + str(x) for x in range(self.sh_nums)]: + # dict is better than list in that case + self.shards[i] = ind_class(self.db_path, sh_name, *args, **kwargs) + self.shards_r['%02x' % i] = self.shards[i] + self.shards_r[i] = self.shards[i] + i += 1 + + if not self.use_make_keys: + self.make_key = self.shards[0].make_key + self.make_key_value = self.shards[0].make_key_value + + self.last_used = 0 + + @property + def storage(self): + st = self.shards[self.last_used].storage + return st + + def __getattr__(self, name): + return getattr(self.shards[self.last_used], name) + + def open_index(self): + for curr in self.shards.values(): + curr.open_index() + + def create_index(self): + for curr in self.shards.values(): + curr.create_index() + + def destroy(self): + for curr in self.shards.values(): + curr.destroy() + + def compact(self): + for curr in self.shards.values(): + curr.compact() + + def reindex(self): + for curr in self.shards.values(): + curr.reindex() + + def all(self, *args, **kwargs): + for curr in self.shards.values(): + for now in curr.all(*args, **kwargs): + yield now + + def get_many(self, *args, **kwargs): + for curr in self.shards.values(): + for now in curr.get_many(*args, **kwargs): + yield now diff --git a/libs/CodernityDB3/storage.py b/libs/CodernityDB3/storage.py new file mode 100644 index 00000000..77bc2475 --- /dev/null +++ b/libs/CodernityDB3/storage.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright 2011-2013 Codernity (http://codernity.com) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import struct +import shutil +import marshal +import io + + +try: + from CodernityDB3 import __version__ +except ImportError: + from .__init__ import __version__ + + +class StorageException(Exception): + pass + + +class DummyStorage(object): + """ + Storage mostly used to fake real storage + """ + + def create(self, *args, **kwargs): + pass + + def open(self, *args, **kwargs): + pass + + def close(self, *args, **kwargs): + pass + + def data_from(self, *args, **kwargs): + pass + + def data_to(self, *args, **kwargs): + pass + + def save(self, *args, **kwargs): + return 0, 0 + + def insert(self, *args, **kwargs): + return self.save(*args, **kwargs) + + def update(self, *args, **kwargs): + return 0, 0 + + def get(self, *args, **kwargs): + return None + + # def compact(self, *args, **kwargs): + # pass + + def fsync(self, *args, **kwargs): + pass + + def flush(self, *args, **kwargs): + pass + + +class IU_Storage(object): + + __version__ = __version__ + + def __init__(self, db_path, name='main'): + if isinstance(db_path, bytes): + db_path = db_path.decode() + if isinstance(name, bytes): + name = name.decode() + + self.db_path = db_path + self.name = name + self._header_size = 100 + + def create(self): + if os.path.exists(os.path.join(self.db_path, self.name + "_stor")): + raise IOError("Storage already exists!") + with io.open(os.path.join(self.db_path, self.name + "_stor"), 'wb') as f: + if isinstance(self.__version__, str): + new_version = self.__version__.encode() + else: + new_version = self.__version__ + f.write(struct.pack(b'10s90s', new_version, b'|||||')) + f.close() + self._f = io.open(os.path.join( + self.db_path, self.name + "_stor"), 'r+b', buffering=0) + self.flush() + self._f.seek(0, 2) + + def open(self): + if not os.path.exists(os.path.join(self.db_path, self.name + "_stor")): + raise IOError("Storage doesn't exists!") + self._f = io.open(os.path.join( + self.db_path, self.name + "_stor"), 'r+b', buffering=0) + self.flush() + self._f.seek(0, 2) + + def destroy(self): + os.unlink(os.path.join(self.db_path, self.name + '_stor')) + + def close(self): + self._f.close() + # self.flush() + # self.fsync() + + def data_from(self, data): + return marshal.loads(data) + + def data_to(self, data): + return marshal.dumps(data) + + def save(self, data): + s_data = self.data_to(data) + self._f.seek(0, 2) + start = self._f.tell() + size = len(s_data) + self._f.write(s_data) + self.flush() + return start, size + + def insert(self, data): + return self.save(data) + + def update(self, data): + return self.save(data) + + def get(self, start, size, status='c'): + if status == 'd': + return None + else: + self._f.seek(start) + return self.data_from(self._f.read(size)) + + def flush(self): + self._f.flush() + + def fsync(self): + os.fsync(self._f.fileno()) + + +# classes for public use, done in this way because of +# generation static files with indexes (_index directory) + + +class Storage(IU_Storage): + pass diff --git a/libs/CodernityDB3/tree_index.py b/libs/CodernityDB3/tree_index.py new file mode 100644 index 00000000..f8236c57 --- /dev/null +++ b/libs/CodernityDB3/tree_index.py @@ -0,0 +1,2117 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright 2011-2013 Codernity (http://codernity.com) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .index import Index, IndexException, DocIdNotFound, ElemNotFound +import struct +import marshal +import os +import io +import shutil +from .storage import IU_Storage +# from ipdb import set_trace + +from CodernityDB3.env import cdb_environment +from CodernityDB3.index import TryReindexException + +if cdb_environment.get('rlock_obj'): + from CodernityDB3 import patch + patch.patch_cache_rr(cdb_environment['rlock_obj']) + +from CodernityDB3.rr_cache import cache1lvl, cache2lvl + +tree_buffer_size = io.DEFAULT_BUFFER_SIZE + +cdb_environment['tree_buffer_size'] = tree_buffer_size + + +MODE_FIRST = 0 +MODE_LAST = 1 + +MOVE_BUFFER_PREV = 0 +MOVE_BUFFER_NEXT = 1 + + +class NodeCapacityException(IndexException): + pass + + +class IU_TreeBasedIndex(Index): + + custom_header = 'from CodernityDB3.tree_index import TreeBasedIndex' + + def __init__(self, db_path, name, key_format='32s', pointer_format='I', + meta_format='32sIIc', node_capacity=10, storage_class=None): + if node_capacity < 3: + raise NodeCapacityException + super(IU_TreeBasedIndex, self).__init__(db_path, name) + self.data_start = self._start_ind + 1 + self.node_capacity = node_capacity + self.flag_format = 'c' + self.elements_counter_format = 'h' + self.pointer_format = pointer_format + self.key_format = key_format + self.meta_format = meta_format + self._count_props() + if not storage_class: + storage_class = IU_Storage + if storage_class and not isinstance(storage_class, str): + storage_class = storage_class.__name__ + self.storage_class = storage_class + self.storage = None + cache = cache1lvl(100) + twolvl_cache = cache2lvl(150) + self._find_key = cache(self._find_key) + self._match_doc_id = cache(self._match_doc_id) +# self._read_single_leaf_record = +# twolvl_cache(self._read_single_leaf_record) + self._find_key_in_leaf = twolvl_cache(self._find_key_in_leaf) + self._read_single_node_key = twolvl_cache(self._read_single_node_key) + self._find_first_key_occurence_in_node = twolvl_cache( + self._find_first_key_occurence_in_node) + self._find_last_key_occurence_in_node = twolvl_cache( + self._find_last_key_occurence_in_node) + self._read_leaf_nr_of_elements = cache(self._read_leaf_nr_of_elements) + self._read_leaf_neighbours = cache(self._read_leaf_neighbours) + self._read_leaf_nr_of_elements_and_neighbours = cache( + self._read_leaf_nr_of_elements_and_neighbours) + self._read_node_nr_of_elements_and_children_flag = cache( + self._read_node_nr_of_elements_and_children_flag) + + def _count_props(self): + """ + Counts dynamic properties for tree, such as all complex formats + """ + self.single_leaf_record_format = self.key_format + self.meta_format + self.single_node_record_format = self.pointer_format + \ + self.key_format + self.pointer_format + self.node_format = self.elements_counter_format + self.flag_format\ + + self.pointer_format + (self.key_format + + self.pointer_format) * self.node_capacity + self.leaf_format = self.elements_counter_format + self.pointer_format * 2\ + + self.single_leaf_record_format * self.node_capacity + self.leaf_heading_format = self.elements_counter_format + \ + self.pointer_format * 2 + self.node_heading_format = self.elements_counter_format + \ + self.flag_format + self.key_size = struct.calcsize('<' + self.key_format) + self.meta_size = struct.calcsize('<' + self.meta_format) + self.single_leaf_record_size = struct.calcsize('<' + self. + single_leaf_record_format) + self.single_node_record_size = struct.calcsize('<' + self. + single_node_record_format) + self.node_size = struct.calcsize('<' + self.node_format) + self.leaf_size = struct.calcsize('<' + self.leaf_format) + self.flag_size = struct.calcsize('<' + self.flag_format) + self.elements_counter_size = struct.calcsize('<' + self. + elements_counter_format) + self.pointer_size = struct.calcsize('<' + self.pointer_format) + self.leaf_heading_size = struct.calcsize( + '<' + self.leaf_heading_format) + self.node_heading_size = struct.calcsize( + '<' + self.node_heading_format) + + def create_index(self): + if os.path.isfile(os.path.join(self.db_path, self.name + '_buck')): + raise IndexException('Already exists') + with io.open(os.path.join(self.db_path, self.name + "_buck"), 'w+b') as f: + props = dict(name=self.name, + flag_format=self.flag_format, + pointer_format=self.pointer_format, + elements_counter_format=self.elements_counter_format, + node_capacity=self.node_capacity, + key_format=self.key_format, + meta_format=self.meta_format, + version=self.__version__, + storage_class=self.storage_class) + f.write(marshal.dumps(props)) + self.buckets = io.open(os.path.join(self.db_path, self.name + + "_buck"), 'r+b', buffering=0) + self._create_storage() + self.buckets.seek(self._start_ind) + self.buckets.write(struct.pack(b' candidate_start: + move_buffer = MOVE_BUFFER_PREV + elif buffer_end < candidate_start + self.single_leaf_record_size: + move_buffer = MOVE_BUFFER_NEXT + else: + move_buffer = None + return self._calculate_key_position(leaf_start, (imin + imax) / 2, 'l'), (imin + imax) / 2, move_buffer + + def _choose_next_candidate_index_in_node(self, node_start, candidate_start, buffer_start, buffer_end, imin, imax): + if buffer_start > candidate_start: + move_buffer = MOVE_BUFFER_PREV + elif buffer_end < candidate_start + self.single_node_record_size: + (self.pointer_size + self.key_size) - 1 + move_buffer = MOVE_BUFFER_NEXT + else: + move_buffer = None + return self._calculate_key_position(node_start, (imin + imax) / 2, 'n'), (imin + imax) / 2, move_buffer + + def _find_key_in_leaf(self, leaf_start, key, nr_of_elements): + if nr_of_elements == 1: + return self._find_key_in_leaf_with_one_element(key, leaf_start)[-5:] + else: + return self._find_key_in_leaf_using_binary_search(key, leaf_start, nr_of_elements)[-5:] + + def _find_key_in_leaf_for_update(self, key, doc_id, leaf_start, nr_of_elements): + if nr_of_elements == 1: + return self._find_key_in_leaf_with_one_element(key, leaf_start, doc_id=doc_id) + else: + return self._find_key_in_leaf_using_binary_search(key, leaf_start, nr_of_elements, mode=MODE_FIRST, doc_id=doc_id) + + def _find_index_of_first_key_equal_or_smaller_key(self, key, leaf_start, nr_of_elements): + if nr_of_elements == 1: + return self._find_key_in_leaf_with_one_element(key, leaf_start, mode=MODE_FIRST, return_closest=True)[:2] + else: + return self._find_key_in_leaf_using_binary_search(key, leaf_start, nr_of_elements, mode=MODE_FIRST, return_closest=True)[:2] + + def _find_index_of_last_key_equal_or_smaller_key(self, key, leaf_start, nr_of_elements): + if nr_of_elements == 1: + return self._find_key_in_leaf_with_one_element(key, leaf_start, mode=MODE_LAST, return_closest=True)[:2] + else: + return self._find_key_in_leaf_using_binary_search(key, leaf_start, nr_of_elements, mode=MODE_LAST, return_closest=True)[:2] + + def _find_index_of_first_key_equal(self, key, leaf_start, nr_of_elements): + if nr_of_elements == 1: + return self._find_key_in_leaf_with_one_element(key, leaf_start, mode=MODE_FIRST)[:2] + else: + return self._find_key_in_leaf_using_binary_search(key, leaf_start, nr_of_elements, mode=MODE_FIRST)[:2] + + def _find_key_in_leaf_with_one_element(self, key, leaf_start, doc_id=None, mode=None, return_closest=False): + curr_key, curr_doc_id, curr_start, curr_size,\ + curr_status = self._read_single_leaf_record(leaf_start, 0) + if key != curr_key: + if return_closest and curr_status != 'd': + return leaf_start, 0 + else: + raise ElemNotFound + else: + if curr_status == 'd': + raise ElemNotFound + elif doc_id is not None and doc_id != curr_doc_id: +# should't happen, crashes earlier on id index + raise DocIdNotFound + else: + return leaf_start, 0, curr_doc_id, curr_key, curr_start, curr_size, curr_status + + def _find_key_in_leaf_using_binary_search(self, key, leaf_start, nr_of_elements, doc_id=None, mode=None, return_closest=False): + """ + Binary search implementation used in all get functions + """ + imin, imax = 0, nr_of_elements - 1 + buffer_start, buffer_end = self._set_buffer_limits() + candidate_start, candidate_index, move_buffer = \ + self._choose_next_candidate_index_in_leaf(leaf_start, + self._calculate_key_position(leaf_start, + (imin + imax) / 2, + 'l'), + buffer_start, + buffer_end, + imin, imax) + while imax != imin and imax > imin: + curr_key, curr_doc_id, curr_start, curr_size, curr_status = self._read_single_leaf_record(leaf_start, + candidate_index) + candidate_start = self._calculate_key_position( + leaf_start, candidate_index, 'l') + if key < curr_key: + if move_buffer == MOVE_BUFFER_PREV: + buffer_start, buffer_end = self._prev_buffer( + buffer_start, buffer_end) + else: # if next chosen element is in current buffer, abort moving to other + move_buffer is None + imax = candidate_index - 1 + candidate_start, candidate_index, move_buffer = self._choose_next_candidate_index_in_leaf(leaf_start, + candidate_start, + buffer_start, + buffer_end, + imin, imax) + elif key == curr_key: + if mode == MODE_LAST: + if move_buffer == MOVE_BUFFER_NEXT: + buffer_start, buffer_end = self._next_buffer( + buffer_start, buffer_end) + else: + move_buffer is None + imin = candidate_index + 1 + candidate_start, candidate_index, move_buffer = self._choose_next_candidate_index_in_leaf(leaf_start, + candidate_start, + buffer_start, + buffer_end, + imin, imax) + else: + if curr_status == 'o': + break + else: + if move_buffer == MOVE_BUFFER_PREV: + buffer_start, buffer_end = self._prev_buffer( + buffer_start, buffer_end) + else: + move_buffer is None + imax = candidate_index + candidate_start, candidate_index, move_buffer = self._choose_next_candidate_index_in_leaf(leaf_start, + candidate_start, + buffer_start, + buffer_end, + imin, imax) + else: + if move_buffer == MOVE_BUFFER_NEXT: + buffer_start, buffer_end = self._next_buffer( + buffer_start, buffer_end) + else: + move_buffer is None + imin = candidate_index + 1 + candidate_start, candidate_index, move_buffer = self._choose_next_candidate_index_in_leaf(leaf_start, + candidate_start, + buffer_start, + buffer_end, + imin, imax) + if imax > imin: + chosen_key_position = candidate_index + else: + chosen_key_position = imax + curr_key, curr_doc_id, curr_start, curr_size, curr_status = self._read_single_leaf_record(leaf_start, + chosen_key_position) + if key != curr_key: + if return_closest: # useful for find all bigger/smaller methods + return leaf_start, chosen_key_position + else: + raise ElemNotFound + if doc_id and doc_id == curr_doc_id and curr_status == 'o': + return leaf_start, chosen_key_position, curr_doc_id, curr_key, curr_start, curr_size, curr_status + else: + if mode == MODE_FIRST and imin < chosen_key_position: # check if there isn't any element with equal key before chosen one + matching_record_index = self._leaf_linear_key_search(key, + self._calculate_key_position(leaf_start, + imin, + 'l'), + imin, + chosen_key_position) + else: + matching_record_index = chosen_key_position + curr_key, curr_doc_id, curr_start, curr_size, curr_status = self._read_single_leaf_record(leaf_start, + matching_record_index) + if curr_status == 'd' and not return_closest: + leaf_start, nr_of_elements, matching_record_index = self._find_existing(key, + matching_record_index, + leaf_start, + nr_of_elements) + curr_key, curr_doc_id, curr_start, curr_size, curr_status = self._read_single_leaf_record(leaf_start, + matching_record_index) + if doc_id is not None and doc_id != curr_doc_id: + leaf_start, nr_of_elements, matching_record_index, curr_status = \ + self._match_doc_id(doc_id, + key, + matching_record_index, + leaf_start, + nr_of_elements) + curr_key, curr_doc_id, curr_start, curr_size, curr_status = \ + self._read_single_leaf_record(leaf_start, + matching_record_index) + return leaf_start, matching_record_index, curr_doc_id, curr_key, curr_start, curr_size, curr_status + + def _find_place_in_leaf(self, key, leaf_start, nr_of_elements): + if nr_of_elements == 1: + return self._find_place_in_leaf_with_one_element(key, leaf_start) + else: + return self._find_place_in_leaf_using_binary_search(key, leaf_start, nr_of_elements) + + def _find_place_in_leaf_with_one_element(self, key, leaf_start): + curr_key, curr_doc_id, curr_start, curr_size,\ + curr_status = self._read_single_leaf_record(leaf_start, 0) + if curr_status == 'd': + return leaf_start, 0, 0, False, True # leaf start, index of new key position, nr of rec to rewrite, full_leaf flag, on_deleted flag + else: + if key < curr_key: + return leaf_start, 0, 1, False, False + else: + return leaf_start, 1, 0, False, False + + def _find_place_in_leaf_using_binary_search(self, key, leaf_start, nr_of_elements): + """ + Binary search implementation used in insert function + """ + imin, imax = 0, nr_of_elements - 1 + buffer_start, buffer_end = self._set_buffer_limits() + candidate_start, candidate_index, move_buffer = self.\ + _choose_next_candidate_index_in_leaf(leaf_start, + self._calculate_key_position(leaf_start, + (imin + imax) / 2, + 'l'), + buffer_start, + buffer_end, + imin, imax) + while imax != imin and imax > imin: + curr_key, curr_doc_id, curr_start, curr_size, curr_status = self._read_single_leaf_record(leaf_start, + candidate_index) + candidate_start = self._calculate_key_position( + leaf_start, candidate_index, 'l') + if key < curr_key: + if move_buffer == MOVE_BUFFER_PREV: + buffer_start, buffer_end = self._prev_buffer( + buffer_start, buffer_end) + else: # if next chosen element is in current buffer, abort moving to other + move_buffer = None + imax = candidate_index - 1 + candidate_start, candidate_index, move_buffer = self._choose_next_candidate_index_in_leaf(leaf_start, + candidate_start, + buffer_start, + buffer_end, + imin, imax) + else: + if move_buffer == MOVE_BUFFER_NEXT: + buffer_start, buffer_end = self._next_buffer( + buffer_start, buffer_end) + else: + move_buffer is None + imin = candidate_index + 1 + candidate_start, candidate_index, move_buffer = self._choose_next_candidate_index_in_leaf(leaf_start, + candidate_start, + buffer_start, + buffer_end, + imin, imax) + if imax < imin and imin < nr_of_elements: + chosen_key_position = imin + else: + chosen_key_position = imax + curr_key, curr_doc_id, curr_start, curr_size, curr_status = self._read_single_leaf_record(leaf_start, + chosen_key_position) + if curr_status == 'd': + return leaf_start, chosen_key_position, 0, False, True + elif key < curr_key: + if chosen_key_position > 0: + curr_key, curr_doc_id, curr_start, curr_size, curr_status = self._read_single_leaf_record(leaf_start, + chosen_key_position - 1) + if curr_start == 'd': + return leaf_start, chosen_key_position - 1, 0, False, True + else: + return leaf_start, chosen_key_position, nr_of_elements - chosen_key_position, (nr_of_elements == self.node_capacity), False + else: + return leaf_start, chosen_key_position, nr_of_elements - chosen_key_position, (nr_of_elements == self.node_capacity), False + else: + if chosen_key_position < nr_of_elements - 1: + curr_key, curr_doc_id, curr_start, curr_size, curr_status = self._read_single_leaf_record(leaf_start, + chosen_key_position + 1) + if curr_start == 'd': + return leaf_start, chosen_key_position + 1, 0, False, True + else: + return leaf_start, chosen_key_position + 1, nr_of_elements - chosen_key_position - 1, (nr_of_elements == self.node_capacity), False + else: + return leaf_start, chosen_key_position + 1, nr_of_elements - chosen_key_position - 1, (nr_of_elements == self.node_capacity), False + + def _set_buffer_limits(self): + pos = self.buckets.tell() + buffer_start = pos - (pos % tree_buffer_size) + return buffer_start, (buffer_start + tree_buffer_size) + + def _find_first_key_occurence_in_node(self, node_start, key, nr_of_elements): + if nr_of_elements == 1: + return self._find_key_in_node_with_one_element(key, node_start, mode=MODE_FIRST) + else: + return self._find_key_in_node_using_binary_search(key, node_start, nr_of_elements, mode=MODE_FIRST) + + def _find_last_key_occurence_in_node(self, node_start, key, nr_of_elements): + if nr_of_elements == 1: + return self._find_key_in_node_with_one_element(key, node_start, mode=MODE_LAST) + else: + return self._find_key_in_node_using_binary_search(key, node_start, nr_of_elements, mode=MODE_LAST) + + def _find_key_in_node_with_one_element(self, key, node_start, mode=None): + l_pointer, curr_key, r_pointer = self._read_single_node_key( + node_start, 0) + if key < curr_key: + return 0, l_pointer + elif key > curr_key: + return 0, r_pointer + else: + if mode == MODE_FIRST: + return 0, l_pointer + elif mode == MODE_LAST: + return 0, r_pointer + else: + raise Exception('Invalid mode declared: set first/last') + + def _find_key_in_node_using_binary_search(self, key, node_start, nr_of_elements, mode=None): + imin, imax = 0, nr_of_elements - 1 + buffer_start, buffer_end = self._set_buffer_limits() + candidate_start, candidate_index, move_buffer = \ + self._choose_next_candidate_index_in_node(node_start, + self._calculate_key_position(node_start, + (imin + imax) / 2, + 'n'), + buffer_start, + buffer_end, + imin, imax) + while imax != imin and imax > imin: + l_pointer, curr_key, r_pointer = self._read_single_node_key( + node_start, candidate_index) + candidate_start = self._calculate_key_position( + node_start, candidate_index, 'n') + if key < curr_key: + if move_buffer == MOVE_BUFFER_PREV: + buffer_start, buffer_end = self._prev_buffer( + buffer_start, buffer_end) + else: # if next chosen element is in current buffer, abort moving to other + move_buffer is None + imax = candidate_index - 1 + candidate_start, candidate_index, move_buffer = self._choose_next_candidate_index_in_node(node_start, + candidate_start, + buffer_start, + buffer_end, + imin, imax) + elif key == curr_key: + if mode == MODE_LAST: + if move_buffer == MOVE_BUFFER_NEXT: + buffer_start, buffer_end = self._next_buffer( + buffer_start, buffer_end) + else: + move_buffer is None + imin = candidate_index + 1 + candidate_start, candidate_index, move_buffer = self._choose_next_candidate_index_in_node(node_start, + candidate_start, + buffer_start, + buffer_end, + imin, imax) + else: + break + else: + if move_buffer == MOVE_BUFFER_NEXT: + buffer_start, buffer_end = self._next_buffer( + buffer_start, buffer_end) + else: + move_buffer is None + imin = candidate_index + 1 + candidate_start, candidate_index, move_buffer = self._choose_next_candidate_index_in_node(node_start, + candidate_start, + buffer_start, + buffer_end, + imin, imax) + + if imax > imin: + chosen_key_position = candidate_index + elif imax < imin and imin < nr_of_elements: + chosen_key_position = imin + else: + chosen_key_position = imax + l_pointer, curr_key, r_pointer = self._read_single_node_key( + node_start, chosen_key_position) + if mode == MODE_FIRST and imin < chosen_key_position: # check if there is no elements with equal key before chosen one + matching_record_index = self._node_linear_key_search(key, + self._calculate_key_position(node_start, + imin, + 'n'), + imin, + chosen_key_position) + else: + matching_record_index = chosen_key_position + l_pointer, curr_key, r_pointer = self._read_single_node_key( + node_start, matching_record_index) + if key < curr_key: + return matching_record_index, l_pointer + elif key > curr_key: + return matching_record_index, r_pointer + else: + if mode == MODE_FIRST: + return matching_record_index, l_pointer + elif mode == MODE_LAST: + return matching_record_index, r_pointer + else: + raise Exception('Invalid mode declared: first/last') + + def _update_leaf_ready_data(self, leaf_start, start_index, new_nr_of_elements, records_to_rewrite): + self.buckets.seek(leaf_start) + self.buckets.write(struct.pack(b' new_leaf_size - 1: + key_moved_to_parent_node = leaf_data[(old_leaf_size - 1) * 5] + elif nr_of_records_to_rewrite == new_leaf_size - 1: + key_moved_to_parent_node = new_data[0] + else: + key_moved_to_parent_node = leaf_data[old_leaf_size * 5] + data_to_write = self._prepare_new_root_data(key_moved_to_parent_node, + left_leaf_start_position, + right_leaf_start_position, + 'l') + if nr_of_records_to_rewrite > half_size: + # key goes to first half + # prepare left leaf data + left_leaf_data = struct.pack(b'<' + str(self.leaf_heading_format + self.single_leaf_record_format + * (self.node_capacity - nr_of_records_to_rewrite)).encode(), + old_leaf_size, + 0, + right_leaf_start_position, + *leaf_data[:-nr_of_records_to_rewrite * 5]) + left_leaf_data += struct.pack( + b'<' + str(self.single_leaf_record_format * ( + nr_of_records_to_rewrite - new_leaf_size + 1)).encode(), + new_data[0], + new_data[1], + new_data[2], + new_data[3], + new_data[4], + *leaf_data[-nr_of_records_to_rewrite * 5:(old_leaf_size - 1) * 5]) + # prepare right leaf_data + right_leaf_data = struct.pack(b'<' + str(self.elements_counter_format + 2 * self.pointer_format + + self.single_leaf_record_format * + new_leaf_size).encode(), + new_leaf_size, + left_leaf_start_position, + 0, + *leaf_data[-new_leaf_size * 5:]) + else: + # key goes to second half + if nr_of_records_to_rewrite: + records_before = leaf_data[old_leaf_size * + 5:-nr_of_records_to_rewrite * 5] + records_after = leaf_data[-nr_of_records_to_rewrite * 5:] + else: + records_before = leaf_data[old_leaf_size * 5:] + records_after = [] + + left_leaf_data = struct.pack( + b'<' + str(self.leaf_heading_format + + self.single_leaf_record_format * old_leaf_size, + old_leaf_size).encode(), + 0, + right_leaf_start_position, + *leaf_data[:old_leaf_size * 5]) + # prepare right leaf_data + right_leaf_data = struct.pack(b'<' + str(self.elements_counter_format + 2 * self.pointer_format + + self.single_leaf_record_format * (new_leaf_size - + nr_of_records_to_rewrite - 1)).encode(), + new_leaf_size, + left_leaf_start_position, + 0, + *records_before) + right_leaf_data += struct.pack( + b'<' + str(self.single_leaf_record_format * ( + nr_of_records_to_rewrite + 1)).encode(), + new_data[0], + new_data[1], + new_data[2], + new_data[3], + new_data[4], + *records_after) + left_leaf_data += (self.node_capacity - + old_leaf_size) * self.single_leaf_record_size * '\x00' + right_leaf_data += blanks + data_to_write += left_leaf_data + data_to_write += right_leaf_data + self.buckets.seek(self._start_ind) + self.buckets.write(struct.pack(b' half_size: # insert key into first half of leaf + self.buckets.seek(self._calculate_key_position(leaf_start, + self.node_capacity - nr_of_records_to_rewrite, + 'l')) + # read all records with key>new_key + data = self.buckets.read( + nr_of_records_to_rewrite * self.single_leaf_record_size) + records_to_rewrite = struct.unpack( + '<' + nr_of_records_to_rewrite * self.single_leaf_record_format, data) + # remove deleted records, if succeded abort spliting + if self._update_if_has_deleted(leaf_start, + records_to_rewrite, + self.node_capacity - + nr_of_records_to_rewrite, + [new_key, new_doc_id, new_start, new_size, new_status]): + return None + key_moved_to_parent_node = records_to_rewrite[ + -new_leaf_size * 5] + # write new leaf at end of file + self.buckets.seek(0, 2) # end of file + new_leaf_start = self.buckets.tell() + # prepare new leaf_data + new_leaf = struct.pack(b'<' + str(self.elements_counter_format + 2 * self.pointer_format + + self.single_leaf_record_format * + new_leaf_size).encode(), + new_leaf_size, + leaf_start, + next_l, + *records_to_rewrite[-new_leaf_size * 5:]) + new_leaf += blanks + # write new leaf + self.buckets.write(new_leaf) + # update old leaf heading + self._update_leaf_size_and_pointers(leaf_start, + old_leaf_size, + prev_l, + new_leaf_start) + # seek position of new key in first half + self.buckets.seek(self._calculate_key_position(leaf_start, + self.node_capacity - nr_of_records_to_rewrite, + 'l')) + # write new key and keys after + self.buckets.write( + struct.pack( + b'<' + str(self.single_leaf_record_format * + (nr_of_records_to_rewrite - new_leaf_size + 1)).encode(), + new_key, + new_doc_id, + new_start, + new_size, + b'o', + *records_to_rewrite[:-new_leaf_size * 5])) + + if next_l: # when next_l is 0 there is no next leaf to update, avoids writing data at 0 position of file + self._update_leaf_prev_pointer( + next_l, new_leaf_start) + +# self._read_single_leaf_record.delete(leaf_start) + self._find_key_in_leaf.delete(leaf_start) + + return new_leaf_start, key_moved_to_parent_node + else: # key goes into second half of leaf ' + # seek half of the leaf + self.buckets.seek(self._calculate_key_position( + leaf_start, old_leaf_size, 'l')) + data = self.buckets.read( + self.single_leaf_record_size * (new_leaf_size - 1)) + records_to_rewrite = struct.unpack('<' + (new_leaf_size - 1) * + self.single_leaf_record_format, data) + # remove deleted records, if succeded abort spliting + if self._update_if_has_deleted(leaf_start, + records_to_rewrite, + old_leaf_size, + [new_key, new_doc_id, new_start, new_size, new_status]): + return None + key_moved_to_parent_node = records_to_rewrite[ + -(new_leaf_size - 1) * 5] + if key_moved_to_parent_node > new_key: + key_moved_to_parent_node = new_key + self.buckets.seek(0, 2) # end of file + new_leaf_start = self.buckets.tell() + # prepare new leaf data + index_of_records_split = nr_of_records_to_rewrite * 5 + if index_of_records_split: + records_before = records_to_rewrite[ + :-index_of_records_split] + records_after = records_to_rewrite[ + -index_of_records_split:] + else: + records_before = records_to_rewrite + records_after = [] + new_leaf = struct.pack(b'<' + str(self.elements_counter_format + 2 * self.pointer_format + + self.single_leaf_record_format * (new_leaf_size - + nr_of_records_to_rewrite - 1)).encode(), + new_leaf_size, + leaf_start, + next_l, + *records_before) + new_leaf += struct.pack( + b'<' + str(self.single_leaf_record_format * + (nr_of_records_to_rewrite + 1)).encode(), + new_key, + new_doc_id, + new_start, + new_size, + b'o', + *records_after) + new_leaf += blanks + self.buckets.write(new_leaf) + self._update_leaf_size_and_pointers(leaf_start, + old_leaf_size, + prev_l, + new_leaf_start) + if next_l: # pren next_l is 0 there is no next leaf to update, avoids writing data at 0 position of file + self._update_leaf_prev_pointer( + next_l, new_leaf_start) + +# self._read_single_leaf_record.delete(leaf_start) + self._find_key_in_leaf.delete(leaf_start) + + return new_leaf_start, key_moved_to_parent_node + + def _update_if_has_deleted(self, leaf_start, records_to_rewrite, start_position, new_record_data): + """ + Checks if there are any deleted elements in data to rewrite and prevent from writing then back. + """ + curr_index = 0 + nr_of_elements = self.node_capacity + records_to_rewrite = list(records_to_rewrite) + for status in records_to_rewrite[4::5]: # remove deleted from list + if status != 'o': + del records_to_rewrite[curr_index * 5:curr_index * 5 + 5] + nr_of_elements -= 1 + else: + curr_index += 1 + # if were deleted dont have to split, just update leaf + if nr_of_elements < self.node_capacity: + data_split_index = 0 + for key in records_to_rewrite[0::5]: + if key > new_record_data[0]: + break + else: + data_split_index += 1 + records_to_rewrite = records_to_rewrite[:data_split_index * 5]\ + + new_record_data\ + + records_to_rewrite[data_split_index * 5:] + self._update_leaf_ready_data(leaf_start, + start_position, + nr_of_elements + 1, + records_to_rewrite), + return True + else: # did not found any deleted records in leaf + return False + + def _prepare_new_root_data(self, root_key, left_pointer, right_pointer, children_flag='n'): + new_root = struct.pack( + b'<' + str(self.node_heading_format + self.single_node_record_format).encode(), + 1, + children_flag, + left_pointer, + root_key, + right_pointer) + new_root += (self.key_size + self.pointer_size) * (self. + node_capacity - 1) * '\x00' + return new_root + + def _create_new_root_from_node(self, node_start, children_flag, nr_of_keys_to_rewrite, new_node_size, old_node_size, new_key, new_pointer): + # reading second half of node + self.buckets.seek(self.data_start + self.node_heading_size) + # read all keys with key>new_key + data = self.buckets.read(self.pointer_size + self. + node_capacity * (self.key_size + self.pointer_size)) + old_node_data = struct.unpack('<' + self.pointer_format + self.node_capacity * + (self.key_format + self.pointer_format), data) + self.buckets.seek(0, 2) # end of file + new_node_start = self.buckets.tell() + if nr_of_keys_to_rewrite == new_node_size: + key_moved_to_root = new_key + # prepare new nodes data + left_node = struct.pack(b'<' + str(self.node_heading_format + self.pointer_format + + old_node_size * (self. + key_format + self.pointer_format)).encode(), + old_node_size, + children_flag, + *old_node_data[:old_node_size * 2 + 1]) + + right_node = struct.pack(b'<' + str(self.node_heading_format + self.pointer_format + + new_node_size * (self. + key_format + self.pointer_format)).encode(), + new_node_size, + children_flag, + new_pointer, + *old_node_data[old_node_size * 2 + 1:]) + elif nr_of_keys_to_rewrite > new_node_size: + key_moved_to_root = old_node_data[old_node_size * 2 - 1] + # prepare new nodes data + if nr_of_keys_to_rewrite == self.node_capacity: + keys_before = old_node_data[:1] + keys_after = old_node_data[1:old_node_size * 2 - 1] + else: + keys_before = old_node_data[:-nr_of_keys_to_rewrite * 2] + keys_after = old_node_data[-( + nr_of_keys_to_rewrite) * 2:old_node_size * 2 - 1] + left_node = struct.pack(b'<' + str(self.node_heading_format + self.pointer_format + + (self.node_capacity - nr_of_keys_to_rewrite) * (self. + key_format + + self.pointer_format)).encode(), + old_node_size, + children_flag, + *keys_before) + left_node += struct.pack( + '<' + (self.key_format + self.pointer_format) * + (nr_of_keys_to_rewrite - new_node_size), + new_key, + new_pointer, + *keys_after) + + right_node = struct.pack(b'<' + str(self.node_heading_format + self.pointer_format + + new_node_size * (self. + key_format + self.pointer_format)).encode(), + new_node_size, + children_flag, + *old_node_data[old_node_size * 2:]) + else: +# 'inserting key into second half of node and creating new root' + key_moved_to_root = old_node_data[old_node_size * 2 + 1] + # prepare new nodes data + left_node = struct.pack(b'<' + str(self.node_heading_format + self.pointer_format + + old_node_size * (self. + key_format + self.pointer_format)).encode(), + old_node_size, + children_flag, + *old_node_data[:old_node_size * 2 + 1]) + if nr_of_keys_to_rewrite: + keys_before = old_node_data[(old_node_size + + 1) * 2:-nr_of_keys_to_rewrite * 2] + keys_after = old_node_data[-nr_of_keys_to_rewrite * 2:] + else: + keys_before = old_node_data[(old_node_size + 1) * 2:] + keys_after = [] + right_node = struct.pack(b'<' + str(self.node_heading_format + self.pointer_format + + (new_node_size - nr_of_keys_to_rewrite - 1) * (self. + key_format + + self.pointer_format)).encode(), + new_node_size, + children_flag, + *keys_before) + right_node += struct.pack( + b'<' + str((nr_of_keys_to_rewrite + 1) * + (self.key_format + self.pointer_format)).encode(), + new_key, + new_pointer, + *keys_after) + new_root = self._prepare_new_root_data(key_moved_to_root, + new_node_start, + new_node_start + self.node_size) + left_node += (self.node_capacity - old_node_size) * \ + (self.key_size + self.pointer_size) * '\x00' + # adding blanks after new node + right_node += (self.node_capacity - new_node_size) * \ + (self.key_size + self.pointer_size) * '\x00' + self.buckets.seek(0, 2) + self.buckets.write(left_node + right_node) + self.buckets.seek(self.data_start) + self.buckets.write(new_root) + + self._read_single_node_key.delete(node_start) + self._read_node_nr_of_elements_and_children_flag.delete(node_start) + return None + + def _split_node(self, node_start, nr_of_keys_to_rewrite, new_key, new_pointer, children_flag, create_new_root=False): + """ + Splits full node in two separate ones, first half of records stays on old position, + second half is written as new leaf at the end of file. + """ + if isinstance(new_key, str): + new_key = new_key.encode() + + half_size = self.node_capacity / 2 + if self.node_capacity % 2 == 0: + old_node_size = new_node_size = half_size + else: + old_node_size = half_size + new_node_size = half_size + 1 + if create_new_root: + self._create_new_root_from_node(node_start, children_flag, nr_of_keys_to_rewrite, new_node_size, old_node_size, new_key, new_pointer) + else: + blanks = (self.node_capacity - new_node_size) * ( + self.key_size + self.pointer_size) * '\x00' + if nr_of_keys_to_rewrite == new_node_size: # insert key into first half of node + # reading second half of node + self.buckets.seek(self._calculate_key_position(node_start, + old_node_size, + 'n') + self.pointer_size) + # read all keys with key>new_key + data = self.buckets.read(nr_of_keys_to_rewrite * + (self.key_size + self.pointer_size)) + old_node_data = struct.unpack('<' + nr_of_keys_to_rewrite * + (self.key_format + self.pointer_format), data) + # write new node at end of file + self.buckets.seek(0, 2) + new_node_start = self.buckets.tell() + # prepare new node_data + new_node = struct.pack(b'<' + str(self.node_heading_format + self.pointer_format + + (self.key_format + + self.pointer_format) * new_node_size).encode(), + new_node_size, + children_flag, + new_pointer, + *old_node_data) + new_node += blanks + # write new node + self.buckets.write(new_node) + # update old node data + self._update_size( + node_start, old_node_size) + + self._read_single_node_key.delete(node_start) + self._read_node_nr_of_elements_and_children_flag.delete( + node_start) + + return new_node_start, new_key + elif nr_of_keys_to_rewrite > half_size: # insert key into first half of node + # seek for first key to rewrite + self.buckets.seek(self._calculate_key_position(node_start, + self.node_capacity - nr_of_keys_to_rewrite, + 'n') + + self.pointer_size) + # read all keys with key>new_key + data = self.buckets.read( + nr_of_keys_to_rewrite * (self.key_size + self.pointer_size)) + old_node_data = struct.unpack( + '<' + nr_of_keys_to_rewrite * (self.key_format + self.pointer_format), data) + key_moved_to_parent_node = old_node_data[-( + new_node_size + 1) * 2] + self.buckets.seek(0, 2) + new_node_start = self.buckets.tell() + # prepare new node_data + new_node = struct.pack(b'<' + str(self.node_heading_format + + self.pointer_format + (self.key_format + + self.pointer_format) * new_node_size).encode(), + new_node_size, + children_flag, + old_node_data[-new_node_size * 2 - 1], + *old_node_data[-new_node_size * 2:]) + new_node += blanks + # write new node + self.buckets.write(new_node) + self._update_size( + node_start, old_node_size) + # seek position of new key in first half + self.buckets.seek(self._calculate_key_position(node_start, + self.node_capacity - nr_of_keys_to_rewrite, + 'n') + + self.pointer_size) + # write new key and keys after + self.buckets.write( + struct.pack( + b'<' + str((self.key_format + self.pointer_format) * + (nr_of_keys_to_rewrite - new_node_size)).encode(), + new_key, + new_pointer, + *old_node_data[:-(new_node_size + 1) * 2])) + + self._read_single_node_key.delete(node_start) + self._read_node_nr_of_elements_and_children_flag.delete( + node_start) + + return new_node_start, key_moved_to_parent_node + else: # key goes into second half + # reading second half of node + self.buckets.seek(self._calculate_key_position(node_start, + old_node_size, + 'n') + + self.pointer_size) + data = self.buckets.read( + new_node_size * (self.key_size + self.pointer_size)) + old_node_data = struct.unpack('<' + new_node_size * + (self.key_format + self.pointer_format), data) + # find key which goes to parent node + key_moved_to_parent_node = old_node_data[0] + self.buckets.seek(0, 2) # end of file + new_node_start = self.buckets.tell() + index_of_records_split = nr_of_keys_to_rewrite * 2 + # prepare new node_data + first_leaf_pointer = old_node_data[1] + old_node_data = old_node_data[2:] + if index_of_records_split: + keys_before = old_node_data[:-index_of_records_split] + keys_after = old_node_data[-index_of_records_split:] + else: + keys_before = old_node_data + keys_after = [] + new_node = struct.pack(b'<' + str(self.node_heading_format + self.pointer_format + + (self.key_format + self.pointer_format) * + (new_node_size - + nr_of_keys_to_rewrite - 1)).encode(), + new_node_size, + children_flag, + first_leaf_pointer, + *keys_before) + new_node += struct.pack(b'<' + str((self.key_format + self.pointer_format) * + (nr_of_keys_to_rewrite + 1)).encode(), + new_key, + new_pointer, + *keys_after) + new_node += blanks + # write new node + self.buckets.write(new_node) + self._update_size(node_start, old_node_size) + + self._read_single_node_key.delete(node_start) + self._read_node_nr_of_elements_and_children_flag.delete( + node_start) + + return new_node_start, key_moved_to_parent_node + + def insert_first_record_into_leaf(self, leaf_start, key, doc_id, start, size, status): + # Fix types + if isinstance(leaf_start, str): + leaf_start = leaf_start.encode() + if isinstance(key, str): + key = key.encode() + if isinstance(doc_id, str): + doc_id = doc_id.encode() + if isinstance(status, str): + status = status.encode() + + self.buckets.seek(leaf_start) + self.buckets.write(struct.pack(b'<' + str(self.elements_counter_format).encode(), + 1)) + self.buckets.seek(leaf_start + self.leaf_heading_size) + self.buckets.write(struct.pack(b'<' + self.single_leaf_record_format.encode(), + key, + doc_id, + start, + size, + status)) + +# self._read_single_leaf_record.delete(leaf_start) + self._find_key_in_leaf.delete(leaf_start) + self._read_leaf_nr_of_elements.delete(leaf_start) + self._read_leaf_nr_of_elements_and_neighbours.delete(leaf_start) + + def _insert_new_record_into_leaf(self, leaf_start, key, doc_id, start, size, status, nodes_stack, indexes): + if isinstance(status, bytes): + status = status.decode() + + nr_of_elements = self._read_leaf_nr_of_elements(leaf_start) + + if nr_of_elements == 0: + self.insert_first_record_into_leaf( + leaf_start, key, doc_id, start, size, status) + return + leaf_start, new_record_position, nr_of_records_to_rewrite, full_leaf, on_deleted\ + = self._find_place_in_leaf(key, leaf_start, nr_of_elements) + if full_leaf: + try: # check if leaf has parent node + leaf_parent_pointer = nodes_stack.pop() + except IndexError: # leaf is a root + leaf_parent_pointer = 0 + split_data = self._split_leaf(leaf_start, + nr_of_records_to_rewrite, + key, + doc_id, + start, + size, + status, + create_new_root=(False if leaf_parent_pointer else True)) + if split_data is not None: # means that split created new root or replaced split with update_if_has_deleted + new_leaf_start_position, key_moved_to_parent_node = split_data + self._insert_new_key_into_node(leaf_parent_pointer, + key_moved_to_parent_node, + leaf_start, + new_leaf_start_position, + nodes_stack, + indexes) + else: # there is a place for record in leaf + self.buckets.seek(leaf_start) + self._update_leaf( + leaf_start, new_record_position, nr_of_elements, nr_of_records_to_rewrite, + on_deleted, key, doc_id, start, size, status) + + def _update_node(self, new_key_position, nr_of_keys_to_rewrite, new_key, new_pointer): + if nr_of_keys_to_rewrite == 0: + self.buckets.seek(new_key_position) + self.buckets.write( + struct.pack(b'<' + str(self.key_format + self.pointer_format).encode(), + new_key, + new_pointer)) + self.flush() + else: + self.buckets.seek(new_key_position) + data = self.buckets.read(nr_of_keys_to_rewrite * ( + self.key_size + self.pointer_size)) + keys_to_rewrite = struct.unpack( + b'<' + str(nr_of_keys_to_rewrite * (self.key_format + self.pointer_format)).encode(), data) + self.buckets.seek(new_key_position) + self.buckets.write( + struct.pack( + b'<' + str((nr_of_keys_to_rewrite + 1) * + (self.key_format + self.pointer_format)).encode(), + new_key, + new_pointer, + *keys_to_rewrite)) + self.flush() + + def _insert_new_key_into_node(self, node_start, new_key, old_half_start, new_half_start, nodes_stack, indexes): + parent_key_index = indexes.pop() + nr_of_elements, children_flag = self._read_node_nr_of_elements_and_children_flag(node_start) + parent_prev_pointer = self._read_single_node_key( + node_start, parent_key_index)[0] + if parent_prev_pointer == old_half_start: # splited child was on the left side of his parent key, must write new key before it + new_key_position = self.pointer_size + self._calculate_key_position(node_start, parent_key_index, 'n') + nr_of_keys_to_rewrite = nr_of_elements - parent_key_index + else: # splited child was on the right side of his parent key, must write new key after it + new_key_position = self.pointer_size + self._calculate_key_position(node_start, parent_key_index + 1, 'n') + nr_of_keys_to_rewrite = nr_of_elements - (parent_key_index + 1) + if nr_of_elements == self.node_capacity: + try: # check if node has parent + node_parent_pointer = nodes_stack.pop() + except IndexError: # node is a root + node_parent_pointer = 0 + new_data = self._split_node(node_start, + nr_of_keys_to_rewrite, + new_key, + new_half_start, + children_flag, + create_new_root=(False if node_parent_pointer else True)) + if new_data: # if not new_data, new root has been created + new_node_start_position, key_moved_to_parent_node = new_data + self._insert_new_key_into_node(node_parent_pointer, + key_moved_to_parent_node, + node_start, + new_node_start_position, + nodes_stack, + indexes) + + self._find_first_key_occurence_in_node.delete(node_start) + self._find_last_key_occurence_in_node.delete(node_start) + else: # there is a empty slot for new key in node + self._update_size(node_start, nr_of_elements + 1) + self._update_node(new_key_position, + nr_of_keys_to_rewrite, + new_key, + new_half_start) + + self._find_first_key_occurence_in_node.delete(node_start) + self._find_last_key_occurence_in_node.delete(node_start) + self._read_single_node_key.delete(node_start) + self._read_node_nr_of_elements_and_children_flag.delete(node_start) + + def _find_leaf_to_insert(self, key): + """ + Traverses tree in search for leaf for insert, remembering parent nodes in path, + looks for last occurence of key if already in tree. + """ + nodes_stack = [self.data_start] + if self.root_flag == 'l': + return nodes_stack, [] + else: + nr_of_elements, curr_child_flag = self._read_node_nr_of_elements_and_children_flag(self.data_start) + curr_index, curr_pointer = self._find_last_key_occurence_in_node( + self.data_start, key, nr_of_elements) + nodes_stack.append(curr_pointer) + indexes = [curr_index] + while(curr_child_flag == 'n'): + nr_of_elements, curr_child_flag = self._read_node_nr_of_elements_and_children_flag(curr_pointer) + curr_index, curr_pointer = self._find_last_key_occurence_in_node(curr_pointer, key, nr_of_elements) + nodes_stack.append(curr_pointer) + indexes.append(curr_index) + return nodes_stack, indexes + # nodes stack contains start addreses of nodes directly above leaf with key, indexes match keys adjacent nodes_stack values (as pointers) + # required when inserting new keys in upper tree levels + + def _find_leaf_with_last_key_occurence(self, key): + if self.root_flag == 'l': + return self.data_start + else: + nr_of_elements, curr_child_flag = self._read_node_nr_of_elements_and_children_flag(self.data_start) + curr_position = self._find_last_key_occurence_in_node( + self.data_start, key, nr_of_elements)[1] + while(curr_child_flag == 'n'): + nr_of_elements, curr_child_flag = self._read_node_nr_of_elements_and_children_flag(curr_position) + curr_position = self._find_last_key_occurence_in_node( + curr_position, key, nr_of_elements)[1] + return curr_position + + def _find_leaf_with_first_key_occurence(self, key): + if self.root_flag == 'l': + return self.data_start + else: + nr_of_elements, curr_child_flag = self._read_node_nr_of_elements_and_children_flag(self.data_start) + curr_position = self._find_first_key_occurence_in_node( + self.data_start, key, nr_of_elements)[1] + while(curr_child_flag == 'n'): + nr_of_elements, curr_child_flag = self._read_node_nr_of_elements_and_children_flag(curr_position) + curr_position = self._find_first_key_occurence_in_node( + curr_position, key, nr_of_elements)[1] + return curr_position + + def _find_key(self, key): + containing_leaf_start = self._find_leaf_with_first_key_occurence(key) + nr_of_elements, prev_leaf, next_leaf = self._read_leaf_nr_of_elements_and_neighbours(containing_leaf_start) + try: + doc_id, l_key, start, size, status = self._find_key_in_leaf( + containing_leaf_start, key, nr_of_elements) + except ElemNotFound: + if next_leaf: + nr_of_elements = self._read_leaf_nr_of_elements(next_leaf) + else: + raise ElemNotFound + doc_id, l_key, start, size, status = self._find_key_in_leaf( + next_leaf, key, nr_of_elements) + return doc_id, l_key, start, size, status + + def _find_key_to_update(self, key, doc_id): + """ + Search tree for key that matches not only given key but also doc_id. + """ + containing_leaf_start = self._find_leaf_with_first_key_occurence(key) + nr_of_elements, prev_leaf, next_leaf = self._read_leaf_nr_of_elements_and_neighbours(containing_leaf_start) + try: + leaf_start, record_index, doc_id, l_key, start, size, status = \ + self._find_key_in_leaf_for_update(key, + doc_id, + containing_leaf_start, + nr_of_elements) + ## print("#####", "_find_key_in_leaf_for_update") # TODO + except ElemNotFound: + if next_leaf: + nr_of_elements = self._read_leaf_nr_of_elements(next_leaf) + else: + raise TryReindexException() + try: + leaf_start, record_index, doc_id, l_key, start, size, status = self._find_key_in_leaf_for_update(key, + doc_id, + next_leaf, + nr_of_elements) + except ElemNotFound: + raise TryReindexException() + return leaf_start, record_index, doc_id, l_key, start, size, status + + def update(self, doc_id, key, u_start=0, u_size=0, u_status='o'): + containing_leaf_start, element_index, old_doc_id, old_key, old_start, old_size, old_status = self._find_key_to_update(key, doc_id) + new_data = (old_doc_id, old_start, old_size, old_status) + if not u_start: + new_data[1] = u_start + if not u_size: + new_data[2] = u_size + if not u_status: + new_data[3] = u_status + self._update_element(containing_leaf_start, element_index, new_data) + + self._find_key.delete(key) + self._match_doc_id.delete(doc_id) + self._find_key_in_leaf.delete(containing_leaf_start, key) + return True + + def delete(self, doc_id, key, start=0, size=0): + containing_leaf_start, element_index = self._find_key_to_update( + key, doc_id)[:2] + print(containing_leaf_start, element_index) + self._delete_element(containing_leaf_start, element_index) + + self._find_key.delete(key) + self._match_doc_id.delete(doc_id) + self._find_key_in_leaf.delete(containing_leaf_start, key) + return True + + def _find_key_many(self, key, limit=1, offset=0): + leaf_with_key = self._find_leaf_with_first_key_occurence(key) + nr_of_elements, prev_leaf, next_leaf = self._read_leaf_nr_of_elements_and_neighbours(leaf_with_key) + try: + leaf_with_key, key_index = self._find_index_of_first_key_equal( + key, leaf_with_key, nr_of_elements) + nr_of_elements, prev_leaf, next_leaf = self._read_leaf_nr_of_elements_and_neighbours(leaf_with_key) + except ElemNotFound: + leaf_with_key = next_leaf + key_index = 0 + nr_of_elements, prev_leaf, next_leaf = self._read_leaf_nr_of_elements_and_neighbours(leaf_with_key) + while offset: + if key_index < nr_of_elements: + curr_key, doc_id, start, size, status = self._read_single_leaf_record( + leaf_with_key, key_index) + if key == curr_key: + if status != 'd': + offset -= 1 + key_index += 1 + else: + return + else: + key_index = 0 + if next_leaf: + leaf_with_key = next_leaf + nr_of_elements, prev_leaf, next_leaf = self._read_leaf_nr_of_elements_and_neighbours(next_leaf) + else: + return + while limit: + if key_index < nr_of_elements: + curr_key, doc_id, start, size, status = self._read_single_leaf_record( + leaf_with_key, key_index) + if key == curr_key: + if status != 'd': + yield doc_id, start, size, status + limit -= 1 + key_index += 1 + else: + return + else: + key_index = 0 + if next_leaf: + leaf_with_key = next_leaf + nr_of_elements, prev_leaf, next_leaf = self._read_leaf_nr_of_elements_and_neighbours(next_leaf) + else: + return + + def _find_key_smaller(self, key, limit=1, offset=0): + leaf_with_key = self._find_leaf_with_first_key_occurence(key) + nr_of_elements, prev_leaf, next_leaf = self._read_leaf_nr_of_elements_and_neighbours(leaf_with_key) + leaf_with_key, key_index = self._find_index_of_first_key_equal_or_smaller_key(key, leaf_with_key, nr_of_elements) + nr_of_elements, prev_leaf, next_leaf = self._read_leaf_nr_of_elements_and_neighbours(leaf_with_key) + curr_key = self._read_single_leaf_record(leaf_with_key, key_index)[0] + if curr_key >= key: + key_index -= 1 + while offset: + if key_index >= 0: + key, doc_id, start, size, status = self._read_single_leaf_record( + leaf_with_key, key_index) + if status != 'd': + offset -= 1 + key_index -= 1 + else: + if prev_leaf: + leaf_with_key = prev_leaf + nr_of_elements, prev_leaf, next_leaf = self._read_leaf_nr_of_elements_and_neighbours(prev_leaf) + key_index = nr_of_elements - 1 + else: + return + while limit: + if key_index >= 0: + key, doc_id, start, size, status = self._read_single_leaf_record( + leaf_with_key, key_index) + if status != 'd': + yield doc_id, key, start, size, status + limit -= 1 + key_index -= 1 + else: + if prev_leaf: + leaf_with_key = prev_leaf + nr_of_elements, prev_leaf, next_leaf = self._read_leaf_nr_of_elements_and_neighbours(prev_leaf) + key_index = nr_of_elements - 1 + else: + return + + def _find_key_equal_and_smaller(self, key, limit=1, offset=0): + leaf_with_key = self._find_leaf_with_last_key_occurence(key) + nr_of_elements, prev_leaf, next_leaf = self._read_leaf_nr_of_elements_and_neighbours(leaf_with_key) + try: + leaf_with_key, key_index = self._find_index_of_last_key_equal_or_smaller_key(key, leaf_with_key, nr_of_elements) + nr_of_elements, prev_leaf, next_leaf = self._read_leaf_nr_of_elements_and_neighbours(leaf_with_key) + except ElemNotFound: + leaf_with_key = prev_leaf + key_index = self._read_leaf_nr_of_elements_and_neighbours( + leaf_with_key)[0] + curr_key = self._read_single_leaf_record(leaf_with_key, key_index)[0] + if curr_key > key: + key_index -= 1 + while offset: + if key_index >= 0: + key, doc_id, start, size, status = self._read_single_leaf_record( + leaf_with_key, key_index) + if status != 'd': + offset -= 1 + key_index -= 1 + else: + if prev_leaf: + leaf_with_key = prev_leaf + nr_of_elements, prev_leaf, next_leaf = self._read_leaf_nr_of_elements_and_neighbours(prev_leaf) + key_index = nr_of_elements - 1 + else: + return + while limit: + if key_index >= 0: + key, doc_id, start, size, status = self._read_single_leaf_record( + leaf_with_key, key_index) + if status != 'd': + yield doc_id, key, start, size, status + limit -= 1 + key_index -= 1 + else: + if prev_leaf: + leaf_with_key = prev_leaf + nr_of_elements, prev_leaf, next_leaf = self._read_leaf_nr_of_elements_and_neighbours(prev_leaf) + key_index = nr_of_elements - 1 + else: + return + + def _find_key_bigger(self, key, limit=1, offset=0): + leaf_with_key = self._find_leaf_with_last_key_occurence(key) + nr_of_elements, prev_leaf, next_leaf = self._read_leaf_nr_of_elements_and_neighbours(leaf_with_key) + try: + leaf_with_key, key_index = self._find_index_of_last_key_equal_or_smaller_key(key, leaf_with_key, nr_of_elements) + nr_of_elements, prev_leaf, next_leaf = self._read_leaf_nr_of_elements_and_neighbours(leaf_with_key) + except ElemNotFound: + key_index = 0 + curr_key = self._read_single_leaf_record(leaf_with_key, key_index)[0] + if curr_key <= key: + key_index += 1 + while offset: + if key_index < nr_of_elements: + curr_key, doc_id, start, size, status = self._read_single_leaf_record( + leaf_with_key, key_index) + if status != 'd': + offset -= 1 + key_index += 1 + else: + key_index = 0 + if next_leaf: + leaf_with_key = next_leaf + nr_of_elements, prev_leaf, next_leaf = self._read_leaf_nr_of_elements_and_neighbours(next_leaf) + else: + return + while limit: + if key_index < nr_of_elements: + curr_key, doc_id, start, size, status = self._read_single_leaf_record( + leaf_with_key, key_index) + if status != 'd': + yield doc_id, curr_key, start, size, status + limit -= 1 + key_index += 1 + else: + key_index = 0 + if next_leaf: + leaf_with_key = next_leaf + nr_of_elements, prev_leaf, next_leaf = self._read_leaf_nr_of_elements_and_neighbours(next_leaf) + else: + return + + def _find_key_equal_and_bigger(self, key, limit=1, offset=0): + leaf_with_key = self._find_leaf_with_first_key_occurence(key) + nr_of_elements, prev_leaf, next_leaf = self._read_leaf_nr_of_elements_and_neighbours(leaf_with_key) + leaf_with_key, key_index = self._find_index_of_first_key_equal_or_smaller_key(key, leaf_with_key, nr_of_elements) + nr_of_elements, prev_leaf, next_leaf = self._read_leaf_nr_of_elements_and_neighbours(leaf_with_key) + curr_key = self._read_single_leaf_record(leaf_with_key, key_index)[0] + if curr_key < key: + key_index += 1 + while offset: + if key_index < nr_of_elements: + curr_key, doc_id, start, size, status = self._read_single_leaf_record( + leaf_with_key, key_index) + if status != 'd': + offset -= 1 + key_index += 1 + else: + key_index = 0 + if next_leaf: + leaf_with_key = next_leaf + nr_of_elements, prev_leaf, next_leaf = self._read_leaf_nr_of_elements_and_neighbours(next_leaf) + else: + return + while limit: + if key_index < nr_of_elements: + curr_key, doc_id, start, size, status = self._read_single_leaf_record( + leaf_with_key, key_index) + if status != 'd': + yield doc_id, curr_key, start, size, status + limit -= 1 + key_index += 1 + else: + key_index = 0 + if next_leaf: + leaf_with_key = next_leaf + nr_of_elements, prev_leaf, next_leaf = self._read_leaf_nr_of_elements_and_neighbours(next_leaf) + else: + return + + def _find_key_between(self, start, end, limit, offset, inclusive_start, inclusive_end): + """ + Returns generator containing all keys withing given interval. + """ + if inclusive_start: + leaf_with_key = self._find_leaf_with_first_key_occurence(start) + nr_of_elements, prev_leaf, next_leaf = self._read_leaf_nr_of_elements_and_neighbours(leaf_with_key) + leaf_with_key, key_index = self._find_index_of_first_key_equal_or_smaller_key(start, leaf_with_key, nr_of_elements) + nr_of_elements, prev_leaf, next_leaf = self._read_leaf_nr_of_elements_and_neighbours(leaf_with_key) + curr_key = self._read_single_leaf_record( + leaf_with_key, key_index)[0] + if curr_key < start: + key_index += 1 + else: + leaf_with_key = self._find_leaf_with_last_key_occurence(start) + nr_of_elements, prev_leaf, next_leaf = self._read_leaf_nr_of_elements_and_neighbours(leaf_with_key) + leaf_with_key, key_index = self._find_index_of_last_key_equal_or_smaller_key(start, leaf_with_key, nr_of_elements) + curr_key, curr_doc_id, curr_start, curr_size, curr_status = self._read_single_leaf_record(leaf_with_key, key_index) + if curr_key <= start: + key_index += 1 + while offset: + if key_index < nr_of_elements: + curr_key, curr_doc_id, curr_start, curr_size, curr_status = self._read_single_leaf_record(leaf_with_key, key_index) + if curr_status != 'd': + offset -= 1 + key_index += 1 + else: + key_index = 0 + if next_leaf: + leaf_with_key = next_leaf + nr_of_elements, prev_leaf, next_leaf = self._read_leaf_nr_of_elements_and_neighbours(next_leaf) + else: + return + while limit: + if key_index < nr_of_elements: + curr_key, curr_doc_id, curr_start, curr_size, curr_status = self._read_single_leaf_record(leaf_with_key, key_index) + if curr_key > end or (curr_key == end and not inclusive_end): + return + elif curr_status != 'd': + yield curr_doc_id, curr_key, curr_start, curr_size, curr_status + limit -= 1 + key_index += 1 + else: + key_index = 0 + if next_leaf: + leaf_with_key = next_leaf + nr_of_elements, prev_leaf, next_leaf = self._read_leaf_nr_of_elements_and_neighbours(next_leaf) + else: + return + + def get(self, key): + ## print("------", type(key)) # TODO + ## print(self) # TODO + k = self.make_key(key) + ## print("K:" * 10, k) # TODO + return self._find_key(k) + + def get_many(self, key, limit=1, offset=0): + return self._find_key_many(self.make_key(key), limit, offset) + + def get_between(self, start, end, limit=1, offset=0, inclusive_start=True, inclusive_end=True): + if start is None: + end = self.make_key(end) + if inclusive_end: + return self._find_key_equal_and_smaller(end, limit, offset) + else: + return self._find_key_smaller(end, limit, offset) + elif end is None: + start = self.make_key(start) + if inclusive_start: + return self._find_key_equal_and_bigger(start, limit, offset) + else: + return self._find_key_bigger(start, limit, offset) + else: + start = self.make_key(start) + end = self.make_key(end) + return self._find_key_between(start, end, limit, offset, inclusive_start, inclusive_end) + + def all(self, limit=-1, offset=0): + """ + Traverses linked list of all tree leaves and returns generator containing all elements stored in index. + """ + if self.root_flag == 'n': + leaf_start = self.data_start + self.node_size + else: + leaf_start = self.data_start + nr_of_elements, prev_leaf, next_leaf = self._read_leaf_nr_of_elements_and_neighbours(leaf_start) + key_index = 0 + while offset: + if key_index < nr_of_elements: + curr_key, doc_id, start, size, status = self._read_single_leaf_record( + leaf_start, key_index) + if status != 'd': + offset -= 1 + key_index += 1 + else: + key_index = 0 + if next_leaf: + leaf_start = next_leaf + nr_of_elements, prev_leaf, next_leaf = self._read_leaf_nr_of_elements_and_neighbours(next_leaf) + else: + return + while limit: + if key_index < nr_of_elements: + curr_key, doc_id, start, size, status = self._read_single_leaf_record( + leaf_start, key_index) + if status != 'd': + yield doc_id, curr_key, start, size, status + limit -= 1 + key_index += 1 + else: + key_index = 0 + if next_leaf: + leaf_start = next_leaf + nr_of_elements, prev_leaf, next_leaf = self._read_leaf_nr_of_elements_and_neighbours(next_leaf) + else: + return + + def make_key(self, key): + raise NotImplementedError() + + def make_key_value(self, data): + raise NotImplementedError() + + def _open_storage(self): + s = globals()[self.storage_class] + if not self.storage: + self.storage = s(self.db_path, self.name) + self.storage.open() + + def _create_storage(self): + s = globals()[self.storage_class] + if not self.storage: + self.storage = s(self.db_path, self.name) + self.storage.create() + + def compact(self, node_capacity=0): + if not node_capacity: + node_capacity = self.node_capacity + + compact_ind = self.__class__( + self.db_path, self.name + '_compact', node_capacity=node_capacity) + compact_ind.create_index() + + gen = self.all() + while True: + try: + doc_id, key, start, size, status = next(gen) + except StopIteration: + break + self.storage._f.seek(start) + value = self.storage._f.read(size) + start_ = compact_ind.storage._f.tell() + compact_ind.storage._f.write(value) + compact_ind.insert(doc_id, key, start_, size, status) + + compact_ind.close_index() + original_name = self.name + # os.unlink(os.path.join(self.db_path, self.name + "_buck")) + self.close_index() + shutil.move(os.path.join(compact_ind.db_path, compact_ind. + name + "_buck"), os.path.join(self.db_path, self.name + "_buck")) + shutil.move(os.path.join(compact_ind.db_path, compact_ind. + name + "_stor"), os.path.join(self.db_path, self.name + "_stor")) + # self.name = original_name + self.open_index() # reload... + self.name = original_name + self._save_params(dict(name=original_name)) + self._fix_params() + self._clear_cache() + return True + + def _fix_params(self): + super(IU_TreeBasedIndex, self)._fix_params() + self._count_props() + + def _clear_cache(self): + self._find_key.clear() + self._match_doc_id.clear() +# self._read_single_leaf_record.clear() + self._find_key_in_leaf.clear() + self._read_single_node_key.clear() + self._find_first_key_occurence_in_node.clear() + self._find_last_key_occurence_in_node.clear() + self._read_leaf_nr_of_elements.clear() + self._read_leaf_neighbours.clear() + self._read_leaf_nr_of_elements_and_neighbours.clear() + self._read_node_nr_of_elements_and_children_flag.clear() + + def close_index(self): + super(IU_TreeBasedIndex, self).close_index() + self._clear_cache() + + +class IU_MultiTreeBasedIndex(IU_TreeBasedIndex): + """ + Class that allows to index more than one key per database record. + + It operates very well on GET/INSERT. It's not optimized for + UPDATE operations (will always readd everything) + """ + + def __init__(self, *args, **kwargs): + super(IU_MultiTreeBasedIndex, self).__init__(*args, **kwargs) + + def insert(self, doc_id, key, start, size, status='o'): + if isinstance(key, (list, tuple)): + key = set(key) + elif not isinstance(key, set): + key = set([key]) + ins = super(IU_MultiTreeBasedIndex, self).insert + for curr_key in key: + ins(doc_id, curr_key, start, size, status) + return True + + def update(self, doc_id, key, u_start, u_size, u_status='o'): + if isinstance(key, (list, tuple)): + key = set(key) + elif not isinstance(key, set): + key = set([key]) + upd = super(IU_MultiTreeBasedIndex, self).update + for curr_key in key: + upd(doc_id, curr_key, u_start, u_size, u_status) + + def delete(self, doc_id, key, start=0, size=0): + if isinstance(key, (list, tuple)): + key = set(key) + elif not isinstance(key, set): + key = set([key]) + delete = super(IU_MultiTreeBasedIndex, self).delete + for curr_key in key: + delete(doc_id, curr_key, start, size) + + def get(self, key): + return super(IU_MultiTreeBasedIndex, self).get(key) + + def make_key_value(self, data): + raise NotImplementedError() + + +# classes for public use, done in this way because of +# generation static files with indexes (_index directory) + + +class TreeBasedIndex(IU_TreeBasedIndex): + pass + + +class MultiTreeBasedIndex(IU_MultiTreeBasedIndex): + """ + It allows to index more than one key for record. (ie. prefix/infix/suffix search mechanizms) + That class is designed to be used in custom indexes. + """ + pass