diff --git a/gluon/cache.py b/gluon/cache.py index 814331cc..57303667 100644 --- a/gluon/cache.py +++ b/gluon/cache.py @@ -44,9 +44,9 @@ except ImportError: have_settings = False try: - import cPickle as pickle + import cPickle as pickle except: - import pickle + import pickle try: import psutil @@ -54,6 +54,7 @@ try: except ImportError: HAVE_PSUTIL = False + def remove_oldest_entries(storage, percentage=90): # compute current memory usage (%) old_mem = psutil.virtual_memory().percent @@ -66,7 +67,8 @@ def remove_oldest_entries(storage, percentage=90): # comute used memory again new_mem = psutil.virtual_memory().percent # if the used memory did not decrease stop - if new_mem >= old_mem: break + if new_mem >= old_mem: + break # net new measurement for memory usage and loop old_mem = new_mem @@ -78,6 +80,7 @@ __all__ = ['Cache', 'lazy_cache'] DEFAULT_TIME_EXPIRE = 300 + class CacheAbstract(object): """ Abstract class for cache implementations. @@ -99,7 +102,7 @@ class CacheAbstract(object): """ cache_stats_name = 'web2py_cache_statistics' - max_ram_utilization = None # percent + max_ram_utilization = None # percent def __init__(self, request=None): """Initializes the object @@ -182,13 +185,14 @@ class CacheInRam(CacheAbstract): self.request = request self.storage = OrderedDict() if HAVE_PSUTIL else {} self.app = request.application if request else '' + def initialize(self): if self.initialized: return else: self.initialized = True self.locker.acquire() - if not self.app in self.meta_storage: + if self.app not in self.meta_storage: self.storage = self.meta_storage[self.app] = \ OrderedDict() if HAVE_PSUTIL else {} self.stats[self.app] = {'hit_total': 0, 'misses': 0} @@ -205,7 +209,7 @@ class CacheInRam(CacheAbstract): else: self._clear(storage, regex) - if not self.app in self.stats: + if self.app not in self.stats: self.stats[self.app] = {'hit_total': 0, 'misses': 0} self.locker.release() @@ -251,8 +255,8 @@ class CacheInRam(CacheAbstract): self.locker.acquire() self.storage[key] = (now, value) self.stats[self.app]['misses'] += 1 - if HAVE_PSUTIL and self.max_ram_utilization!=None and random.random()<0.10: - remove_oldest_entries(self.storage, percentage = self.max_ram_utilization) + if HAVE_PSUTIL and self.max_ram_utilization is not None and random.random() < 0.10: + remove_oldest_entries(self.storage, percentage=self.max_ram_utilization) self.locker.release() return value @@ -292,14 +296,15 @@ class CacheOnDisk(CacheAbstract): self.folder = folder self.key_filter_in = lambda key: key self.key_filter_out = lambda key: key - self.file_lock_time_wait = file_lock_time_wait # How long we should wait before retrying to lock a file held by another process + self.file_lock_time_wait = file_lock_time_wait + # How long we should wait before retrying to lock a file held by another process # We still need a mutex for each file as portalocker only blocks other processes self.file_locks = defaultdict(thread.allocate_lock) - # Make sure we use valid filenames. if sys.platform == "win32": import base64 + def key_filter_in_windows(key): """ Windows doesn't allow \ / : * ? "< > | in filenames. @@ -316,7 +321,6 @@ class CacheOnDisk(CacheAbstract): self.key_filter_in = key_filter_in_windows self.key_filter_out = key_filter_out_windows - def wait_portalock(self, val_file): """ Wait for the process file lock. @@ -328,15 +332,12 @@ class CacheOnDisk(CacheAbstract): except: time.sleep(self.file_lock_time_wait) - def acquire(self, key): self.file_locks[key].acquire() - def release(self, key): self.file_locks[key].release() - def __setitem__(self, key, value): key = self.key_filter_in(key) val_file = recfile.open(key, mode='wb', path=self.folder) @@ -344,7 +345,6 @@ class CacheOnDisk(CacheAbstract): pickle.dump(value, val_file, pickle.HIGHEST_PROTOCOL) val_file.close() - def __getitem__(self, key): key = self.key_filter_in(key) try: @@ -357,12 +357,10 @@ class CacheOnDisk(CacheAbstract): val_file.close() return value - def __contains__(self, key): key = self.key_filter_in(key) return (key in self.file_locks) or recfile.exists(key, path=self.folder) - def __delitem__(self, key): key = self.key_filter_in(key) try: @@ -370,13 +368,11 @@ class CacheOnDisk(CacheAbstract): except IOError: raise KeyError - def __iter__(self): for dirpath, dirnames, filenames in os.walk(self.folder): for filename in filenames: yield self.key_filter_out(filename) - def safe_apply(self, key, function, default_value=None): """ Safely apply a function to the value of a key in storage and set @@ -403,25 +399,21 @@ class CacheOnDisk(CacheAbstract): val_file.close() return new_value - def keys(self): return list(self.__iter__()) - def get(self, key, default=None): try: return self[key] except KeyError: return default - def __init__(self, request=None, folder=None): self.initialized = False self.request = request self.folder = folder self.storage = None - def initialize(self): if self.initialized: return @@ -440,7 +432,6 @@ class CacheOnDisk(CacheAbstract): self.storage = CacheOnDisk.PersistentStorage(folder) - def __call__(self, key, f, time_expire=DEFAULT_TIME_EXPIRE): self.initialize() @@ -487,7 +478,6 @@ class CacheOnDisk(CacheAbstract): self.storage.release(key) return value - def clear(self, regex=None): self.initialize() storage = self.storage @@ -504,7 +494,6 @@ class CacheOnDisk(CacheAbstract): pass storage.release(key) - def increment(self, key, value=1): self.initialize() self.storage.acquire(key) @@ -513,7 +502,6 @@ class CacheOnDisk(CacheAbstract): return value - class CacheAction(object): def __init__(self, func, key, time_expire, cache, cache_model): self.__name__ = func.__name__ @@ -572,9 +560,9 @@ class Cache(object): logger.warning('no cache.disk (AttributeError)') def action(self, time_expire=DEFAULT_TIME_EXPIRE, cache_model=None, - prefix=None, session=False, vars=True, lang=True, - user_agent=False, public=True, valid_statuses=None, - quick=None): + prefix=None, session=False, vars=True, lang=True, + user_agent=False, public=True, valid_statuses=None, + quick=None): """Better fit for caching an action Warning: @@ -602,6 +590,7 @@ class Cache(object): """ from gluon import current from gluon.http import HTTP + def wrap(func): def wrapped_f(): if current.request.env.request_method != 'GET': @@ -621,13 +610,14 @@ class Cache(object): cache_control = 'max-age=%(time_expire)s, s-maxage=%(time_expire)s' % dict(time_expire=time_expire) if not session_ and public_: cache_control += ', public' - expires = (current.request.utcnow + datetime.timedelta(seconds=time_expire)).strftime('%a, %d %b %Y %H:%M:%S GMT') + expires = (current.request.utcnow + datetime.timedelta(seconds=time_expire) + ).strftime('%a, %d %b %Y %H:%M:%S GMT') else: cache_control += ', private' expires = 'Fri, 01 Jan 1990 00:00:00 GMT' if cache_model: - #figure out the correct cache key + # figure out the correct cache key cache_key = [current.request.env.path_info, current.response.view] if session_: cache_key.append(current.response.session_id) @@ -644,28 +634,28 @@ class Cache(object): if prefix: cache_key = prefix + cache_key try: - #action returns something - rtn = cache_model(cache_key, lambda : func(), time_expire=time_expire) + # action returns something + rtn = cache_model(cache_key, lambda: func(), time_expire=time_expire) http, status = None, current.response.status except HTTP, e: - #action raises HTTP (can still be valid) - rtn = cache_model(cache_key, lambda : e.body, time_expire=time_expire) + # action raises HTTP (can still be valid) + rtn = cache_model(cache_key, lambda: e.body, time_expire=time_expire) http, status = HTTP(e.status, rtn, **e.headers), e.status else: - #action raised a generic exception + # action raised a generic exception http = None else: - #no server-cache side involved + # no server-cache side involved try: - #action returns something + # action returns something rtn = func() http, status = None, current.response.status except HTTP, e: - #action raises HTTP (can still be valid) + # action raises HTTP (can still be valid) status = e.status http = HTTP(e.status, e.body, **e.headers) else: - #action raised a generic exception + # action raised a generic exception http = None send_headers = False if http and isinstance(valid_statuses, list): @@ -675,15 +665,13 @@ class Cache(object): if str(status)[0] in '123': send_headers = True if send_headers: - headers = { - 'Pragma' : None, - 'Expires' : expires, - 'Cache-Control' : cache_control - } + headers = {'Pragma': None, + 'Expires': expires, + 'Cache-Control': cache_control} current.response.headers.update(headers) if cache_model and not send_headers: - #we cached already the value, but the status is not valid - #so we need to delete the cached value + # we cached already the value, but the status is not valid + # so we need to delete the cached value cache_model(cache_key, None) if http: if send_headers: @@ -740,8 +728,7 @@ class Cache(object): allow replacing cache.ram with cache.with_prefix(cache.ram,'prefix') it will add prefix to all the cache keys used. """ - return lambda key, f, time_expire=DEFAULT_TIME_EXPIRE, prefix=prefix:\ - cache_model(prefix + key, f, time_expire) + return lambda key, f, time_expire=DEFAULT_TIME_EXPIRE, prefix=prefix: cache_model(prefix + key, f, time_expire) def lazy_cache(key=None, time_expire=None, cache_model='ram'): diff --git a/gluon/tests/test_cache.py b/gluon/tests/test_cache.py index 68dd743e..31ccd73c 100644 --- a/gluon/tests/test_cache.py +++ b/gluon/tests/test_cache.py @@ -37,10 +37,11 @@ def tearDownModule(): pass - class TestCache(unittest.TestCase): - def testCacheInRam(self): + # TODO: test_CacheAbstract(self): + + def test_CacheInRam(self): # defaults to mode='http' cache = CacheInRam() @@ -53,22 +54,21 @@ class TestCache(unittest.TestCase): cache.clear() self.assertEqual(cache('a', lambda: 3, 100), 3) self.assertEqual(cache('a', lambda: 4, 0), 4) - #test singleton behaviour + # test singleton behaviour cache = CacheInRam() cache.clear() self.assertEqual(cache('a', lambda: 3, 100), 3) self.assertEqual(cache('a', lambda: 4, 0), 4) - #test key deletion + # test key deletion cache('a', None) self.assertEqual(cache('a', lambda: 5, 100), 5) - #test increment + # test increment self.assertEqual(cache.increment('a'), 6) self.assertEqual(cache('a', lambda: 1, 100), 6) cache.increment('b') self.assertEqual(cache('b', lambda: 'x', 100), 1) - - def testCacheOnDisk(self): + def test_CacheOnDisk(self): # defaults to mode='http' s = Storage({'application': 'admin', @@ -83,30 +83,36 @@ class TestCache(unittest.TestCase): cache.clear() self.assertEqual(cache('a', lambda: 3, 100), 3) self.assertEqual(cache('a', lambda: 4, 0), 4) - #test singleton behaviour + # test singleton behaviour cache = CacheOnDisk(s) cache.clear() self.assertEqual(cache('a', lambda: 3, 100), 3) self.assertEqual(cache('a', lambda: 4, 0), 4) - #test key deletion + # test key deletion cache('a', None) self.assertEqual(cache('a', lambda: 5, 100), 5) - #test increment + # test increment self.assertEqual(cache.increment('a'), 6) self.assertEqual(cache('a', lambda: 1, 100), 6) cache.increment('b') self.assertEqual(cache('b', lambda: 'x', 100), 1) - def testCacheWithPrefix(self): + # TODO: def test_CacheAction(self): + + # TODO: def test_Cache(self): + + # TODO: def test_lazy_cache(self): + + def test_CacheWithPrefix(self): s = Storage({'application': 'admin', 'folder': 'applications/admin'}) cache = Cache(s) - prefix = cache.with_prefix(cache.ram,'prefix') + prefix = cache.with_prefix(cache.ram, 'prefix') self.assertEqual(prefix('a', lambda: 1, 0), 1) self.assertEqual(prefix('a', lambda: 2, 100), 1) self.assertEqual(cache.ram('prefixa', lambda: 2, 100), 1) - def testRegex(self): + def test_Regex(self): cache = CacheInRam() self.assertEqual(cache('a1', lambda: 1, 0), 1) self.assertEqual(cache('a2', lambda: 2, 100), 2) @@ -114,7 +120,7 @@ class TestCache(unittest.TestCase): self.assertEqual(cache('a1', lambda: 2, 0), 2) self.assertEqual(cache('a2', lambda: 3, 100), 3) - def testDALcache(self): + def test_DALcache(self): s = Storage({'application': 'admin', 'folder': 'applications/admin'}) cache = Cache(s)