diff --git a/VERSION b/VERSION index e39490ac..b97b7bc2 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.1.0 (2012-10-10 09:18:57) dev +Version 2.1.0 (2012-10-10 11:01:05) dev diff --git a/gluon/cache.py b/gluon/cache.py index 8d9c1c63..7f974ff1 100644 --- a/gluon/cache.py +++ b/gluon/cache.py @@ -11,7 +11,7 @@ Basic caching classes and methods - Cache - The generic caching object interfacing with the others - CacheInRam - providing caching in ram -- CacheInDisk - provides caches on disk +- CacheOnDisk - provides caches on disk Memcache is also available via a different module (see gluon.contrib.memcache) @@ -130,22 +130,27 @@ class CacheInRam(CacheAbstract): meta_storage = {} def __init__(self, request=None): - self.locker.acquire() + self.initialized = False self.request = request + + def initialize(self): + if self.initialized: return + else: self.initialized = True + self.locker.acquire() + request = self.request if request: app = request.application else: app = '' if not app in self.meta_storage: - self.storage = self.meta_storage[app] = {CacheAbstract.cache_stats_name: { - 'hit_total': 0, - 'misses': 0, - }} + self.storage = self.meta_storage[app] = { + CacheAbstract.cache_stats_name: {'hit_total': 0, 'misses': 0}} else: self.storage = self.meta_storage[app] self.locker.release() def clear(self, regex=None): + self.initialize() self.locker.acquire() storage = self.storage if regex is None: @@ -154,10 +159,7 @@ class CacheInRam(CacheAbstract): self._clear(storage, regex) if not CacheAbstract.cache_stats_name in storage.keys(): - storage[CacheAbstract.cache_stats_name] = { - 'hit_total': 0, - 'misses': 0, - } + storage[CacheAbstract.cache_stats_name] = {'hit_total': 0,'misses': 0} self.locker.release() @@ -172,6 +174,7 @@ class CacheInRam(CacheAbstract): 3) would work unless we deepcopy no storage and retrival which would make things slow. Anyway. You can deepcopy explicitly in the function generating the value to be cached. """ + self.initialize() dt = time_expire now = time.time() @@ -200,6 +203,7 @@ class CacheInRam(CacheAbstract): return value def increment(self, key, value=1): + self.initialize() self.locker.acquire() try: if key in self.storage: @@ -226,58 +230,66 @@ class CacheOnDisk(CacheAbstract): Values stored in disk cache must be pickable. """ - speedup_checks = set() + def _close_shelve_and_unlock(self): + try: + if self.storage: + self.storage.close() + finally: + if self.locker and self.locked: + portalocker.unlock(self.locker) + self.locker.close() + self.locked = False - def _open_shelf_with_lock(self): + def _open_shelve_and_lock(self): """Open and return a shelf object, obtaining an exclusive lock on self.locker first. Replaces the close method of the returned shelf instance with one that releases the lock upon closing.""" - def _close(self): - try: - shelve.Shelf.close(self) - finally: - portalocker.unlock(self.locker) - self.locker.close() - - storage, locker, locker_locked = None, None, False + + storage = None + locker = None + locked = False try: - locker = open(self.locker_name, 'a') + locker = locker = open(self.locker_name, 'a') portalocker.lock(locker, portalocker.LOCK_EX) - locker_locked = True - storage = shelve.open(self.shelve_name) - storage.close = _close.__get__(storage, shelve.Shelf) - storage.locker = locker - except Exception: - logger.error('corrupted cache file %s, will try to delete and recreate it!' % (self.shelve_name)) + locked = True + try: + storage = shelve.open(self.shelve_name) + except: + logger.error('corrupted cache file %s, will try rebuild it' \ + % (self.shelve_name)) + storage = None + if not storage and os.path.exists(self.shelve_name): + os.unlink(self.shelve_name) + storage = shelve.open(self.shelve_name) + if not CacheAbstract.cache_stats_name in storage.keys(): + storage[CacheAbstract.cache_stats_name] = {'hit_total':0, 'misses': 0} + storage.sync() + except Exception, e: if storage: storage.close() storage = None - - try: - os.unlink(self.shelve_name) - storage = shelve.open(self.shelve_name) - storage.close = _close.__get__(storage, shelve.Shelf) - storage.locker = locker - if not CacheAbstract.cache_stats_name in storage.keys(): - storage[CacheAbstract.cache_stats_name] = { - 'hit_total': 0, - 'misses': 0, - } - storage.sync() - except (IOError, OSError): - logger.warn('unable to delete and recreate cache file %s' % self.shelve_name) - if storage: - storage.close() - storage = None - if locker_locked: - portalocker.unlock(locker) - if locker: - locker.close() + if locked: + portalocker.unlock(locker) + locker.close() + locked = False + raise RuntimeError, 'unable to create/re-create cache file %s' % self.shelve_name + self.locker = locker + self.locked = locked + self.storage = storage return storage - def __init__(self, request, folder=None): + def __init__(self, request=None, folder=None): + self.initialized = False self.request = request + self.folder = folder + self.initialize() + + def initialize(self): + if self.initialized: return + else: self.initialized = True + folder = self.folder + request = self.request # Lets test if the cache folder exists, if not # we are going to create it @@ -291,93 +303,54 @@ class CacheOnDisk(CacheAbstract): self.locker_name = os.path.join(folder,'cache.lock') self.shelve_name = os.path.join(folder,'cache.shelve') - speedup_key = (folder,CacheAbstract.cache_stats_name) - if not speedup_key in self.speedup_checks or \ - not os.path.exists(self.shelve_name): - try: - storage = self._open_shelf_with_lock() - try: - if not CacheAbstract.cache_stats_name in storage: - storage[CacheAbstract.cache_stats_name] = { - 'hit_total': 0, - 'misses': 0, - } - storage.sync() - finally: - storage.close() - self.speedup_checks.add(speedup_key) - except ImportError: - pass # no module _bsddb, ignoring exception now so it makes a ticket only if used - def clear(self, regex=None): - storage = self._open_shelf_with_lock() + self.initialize() + storage = self._open_shelve_and_lock() try: if regex is None: storage.clear() else: self._clear(storage, regex) - if not CacheAbstract.cache_stats_name in storage.keys(): - storage[CacheAbstract.cache_stats_name] = { - 'hit_total': 0, - 'misses': 0, - } storage.sync() finally: - storage.close() + self._close_shelve_and_unlock() def __call__(self, key, f, time_expire = DEFAULT_TIME_EXPIRE): + self.initialize() dt = time_expire - - storage = self._open_shelf_with_lock() + storage = self._open_shelve_and_lock() try: item = storage.get(key, None) + storage[CacheAbstract.cache_stats_name]['hit_total'] += 1 if item and f is None: del storage[key] - - storage[CacheAbstract.cache_stats_name] = { - 'hit_total': storage[CacheAbstract.cache_stats_name]['hit_total'] + 1, - 'misses': storage[CacheAbstract.cache_stats_name]['misses'] - } - - storage.sync() + storage.sync() + now = time.time() + if f is None: + value = None + elif item and (dt is None or item[0] > now - dt): + value = item[1] + else: + value = f() + storage[key] = (now, value) + storage[CacheAbstract.cache_stats_name]['misses']+=1 + storage.sync() finally: - if storage: - storage.close() - - now = time.time() - if f is None: - return None - if item and (dt is None or item[0] > now - dt): - return item[1] - value = f() - - storage = self._open_shelf_with_lock() - try: - storage[key] = (now, value) - - storage[CacheAbstract.cache_stats_name] = { - 'hit_total': storage[CacheAbstract.cache_stats_name]['hit_total'], - 'misses': storage[CacheAbstract.cache_stats_name]['misses'] + 1 - } - - storage.sync() - finally: - if storage: - storage.close() + self._close_shelve_and_unlock() return value def increment(self, key, value=1): - storage = self._open_shelf_with_lock() + self.initialize() + storage = self._open_shelve_and_lock() try: if key in storage: value = storage[key][1] + value storage[key] = (time.time(), value) storage.sync() finally: - if storage: - storage.close() + self._close_shelve_and_unlock() return value class CacheAction(object): @@ -423,10 +396,9 @@ class Cache(object): the global request object """ # GAE will have a special caching - if have_settings and settings.global_settings.web2py_runtime_gae: from contrib.gae_memcache import MemcacheClient - self.ram=self.disk=MemcacheClient(request) + self.ram = self.disk = MemcacheClient(request) else: # Otherwise use ram (and try also disk) self.ram = CacheInRam(request) @@ -489,7 +461,6 @@ class Cache(object): cache_model(prefix + key, f, time_expire) - def lazy_cache(key=None,time_expire=None,cache_model='ram'): """ can be used to cache any function including in modules, diff --git a/gluon/compileapp.py b/gluon/compileapp.py index 1202a2f2..f131b33b 100644 --- a/gluon/compileapp.py +++ b/gluon/compileapp.py @@ -387,6 +387,7 @@ def build_environment(request, response, session, store_current=True): t = environment['T'] = translator(request) c = environment['cache'] = Cache(request) + if store_current: current.globalenv = environment current.request = request diff --git a/gluon/main.py b/gluon/main.py index 0b68753d..89cc2d9c 100644 --- a/gluon/main.py +++ b/gluon/main.py @@ -88,7 +88,7 @@ from contenttype import contenttype from dal import BaseAdapter from settings import global_settings from validators import CRYPT -from cache import Cache +from cache import CacheInRam from html import URL, xmlescape from utils import is_valid_ip_address from rewrite import load, url_in, thread as rwthread, \ @@ -148,7 +148,7 @@ def get_client(env): def copystream_progress(request, chunk_size= 10**5): """ copies request.env.wsgi_input into request.body - and stores progress upload status in cache.ram + and stores progress upload status in cache_ram X-Progress-ID:length and X-Progress-ID:uploaded """ env = request.env @@ -164,16 +164,16 @@ def copystream_progress(request, chunk_size= 10**5): copystream(source, dest, size, chunk_size) return dest cache_key = 'X-Progress-ID:'+request.vars['X-Progress-ID'] - cache = Cache(request) - cache.ram(cache_key+':length', lambda: size, 0) - cache.ram(cache_key+':uploaded', lambda: 0, 0) + cache_ram = CacheInRam(request) # same as cache.ram because meta_storage + cache_ram(cache_key+':length', lambda: size, 0) + cache_ram(cache_key+':uploaded', lambda: 0, 0) while size > 0: if size < chunk_size: data = source.read(size) - cache.ram.increment(cache_key+':uploaded', size) + cache_ram.increment(cache_key+':uploaded', size) else: data = source.read(chunk_size) - cache.ram.increment(cache_key+':uploaded', chunk_size) + cache_ram.increment(cache_key+':uploaded', chunk_size) length = len(data) if length > size: (data, length) = (data[:size], size) @@ -184,8 +184,8 @@ def copystream_progress(request, chunk_size= 10**5): if length < chunk_size: break dest.seek(0) - cache.ram(cache_key+':length', None) - cache.ram(cache_key+':uploaded', None) + cache_ram(cache_key+':length', None) + cache_ram(cache_key+':uploaded', None) return dest