dinislage/web2py
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

cache.py PEP8 enhancement

Browse Source
This commit is contained in:
Hardirc
2016-03-26 13:11:27 -04:00
parent 060aeff867
commit 58a8ba067c
1 changed files with 37 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files
gluon/cache.py
+37 -50
View File
@@ -44,9 +44,9 @@ except ImportError:
have_settings = False have_settings = False
try: try:
import cPickle as pickle import cPickle as pickle
except: except:
import pickle import pickle
try: try:
import psutil import psutil
@@ -54,6 +54,7 @@ try:
except ImportError: except ImportError:
HAVE_PSUTIL = False HAVE_PSUTIL = False
def remove_oldest_entries(storage, percentage=90): def remove_oldest_entries(storage, percentage=90):
# compute current memory usage (%) # compute current memory usage (%)
old_mem = psutil.virtual_memory().percent old_mem = psutil.virtual_memory().percent
@@ -66,7 +67,8 @@ def remove_oldest_entries(storage, percentage=90):
# comute used memory again # comute used memory again
new_mem = psutil.virtual_memory().percent new_mem = psutil.virtual_memory().percent
# if the used memory did not decrease stop # if the used memory did not decrease stop
if new_mem >= old_mem: break if new_mem >= old_mem:
break
# net new measurement for memory usage and loop # net new measurement for memory usage and loop
old_mem = new_mem old_mem = new_mem
@@ -78,6 +80,7 @@ __all__ = ['Cache', 'lazy_cache']
DEFAULT_TIME_EXPIRE = 300 DEFAULT_TIME_EXPIRE = 300
class CacheAbstract(object): class CacheAbstract(object):
""" """
Abstract class for cache implementations. Abstract class for cache implementations.
@@ -99,7 +102,7 @@ class CacheAbstract(object):
""" """
cache_stats_name = 'web2py_cache_statistics' cache_stats_name = 'web2py_cache_statistics'
max_ram_utilization = None # percent max_ram_utilization = None # percent
def __init__(self, request=None): def __init__(self, request=None):
"""Initializes the object """Initializes the object
@@ -182,13 +185,14 @@ class CacheInRam(CacheAbstract):
self.request = request self.request = request
self.storage = OrderedDict() if HAVE_PSUTIL else {} self.storage = OrderedDict() if HAVE_PSUTIL else {}
self.app = request.application if request else '' self.app = request.application if request else ''
def initialize(self): def initialize(self):
if self.initialized: if self.initialized:
return return
else: else:
self.initialized = True self.initialized = True
self.locker.acquire() self.locker.acquire()
if not self.app in self.meta_storage: if self.app not in self.meta_storage:
self.storage = self.meta_storage[self.app] = \ self.storage = self.meta_storage[self.app] = \
OrderedDict() if HAVE_PSUTIL else {} OrderedDict() if HAVE_PSUTIL else {}
self.stats[self.app] = {'hit_total': 0, 'misses': 0} self.stats[self.app] = {'hit_total': 0, 'misses': 0}
@@ -205,7 +209,7 @@ class CacheInRam(CacheAbstract):
else: else:
self._clear(storage, regex) self._clear(storage, regex)
if not self.app in self.stats: if self.app not in self.stats:
self.stats[self.app] = {'hit_total': 0, 'misses': 0} self.stats[self.app] = {'hit_total': 0, 'misses': 0}
self.locker.release() self.locker.release()
@@ -251,8 +255,8 @@ class CacheInRam(CacheAbstract):
self.locker.acquire() self.locker.acquire()
self.storage[key] = (now, value) self.storage[key] = (now, value)
self.stats[self.app]['misses'] += 1 self.stats[self.app]['misses'] += 1
if HAVE_PSUTIL and self.max_ram_utilization!=None and random.random()<0.10: if HAVE_PSUTIL and self.max_ram_utilization is not None and random.random() < 0.10:
remove_oldest_entries(self.storage, percentage = self.max_ram_utilization) remove_oldest_entries(self.storage, percentage=self.max_ram_utilization)
self.locker.release() self.locker.release()
return value return value
@@ -292,14 +296,15 @@ class CacheOnDisk(CacheAbstract):
self.folder = folder self.folder = folder
self.key_filter_in = lambda key: key self.key_filter_in = lambda key: key
self.key_filter_out = lambda key: key self.key_filter_out = lambda key: key
self.file_lock_time_wait = file_lock_time_wait # How long we should wait before retrying to lock a file held by another process self.file_lock_time_wait = file_lock_time_wait
# How long we should wait before retrying to lock a file held by another process
# We still need a mutex for each file as portalocker only blocks other processes # We still need a mutex for each file as portalocker only blocks other processes
self.file_locks = defaultdict(thread.allocate_lock) self.file_locks = defaultdict(thread.allocate_lock)
# Make sure we use valid filenames. # Make sure we use valid filenames.
if sys.platform == "win32": if sys.platform == "win32":
import base64 import base64
def key_filter_in_windows(key): def key_filter_in_windows(key):
""" """
Windows doesn't allow \ / : * ? "< > | in filenames. Windows doesn't allow \ / : * ? "< > | in filenames.
@@ -316,7 +321,6 @@ class CacheOnDisk(CacheAbstract):
self.key_filter_in = key_filter_in_windows self.key_filter_in = key_filter_in_windows
self.key_filter_out = key_filter_out_windows self.key_filter_out = key_filter_out_windows
def wait_portalock(self, val_file): def wait_portalock(self, val_file):
""" """
Wait for the process file lock. Wait for the process file lock.
@@ -328,15 +332,12 @@ class CacheOnDisk(CacheAbstract):
except: except:
time.sleep(self.file_lock_time_wait) time.sleep(self.file_lock_time_wait)
def acquire(self, key): def acquire(self, key):
self.file_locks[key].acquire() self.file_locks[key].acquire()
def release(self, key): def release(self, key):
self.file_locks[key].release() self.file_locks[key].release()
def __setitem__(self, key, value): def __setitem__(self, key, value):
key = self.key_filter_in(key) key = self.key_filter_in(key)
val_file = recfile.open(key, mode='wb', path=self.folder) val_file = recfile.open(key, mode='wb', path=self.folder)
@@ -344,7 +345,6 @@ class CacheOnDisk(CacheAbstract):
pickle.dump(value, val_file, pickle.HIGHEST_PROTOCOL) pickle.dump(value, val_file, pickle.HIGHEST_PROTOCOL)
val_file.close() val_file.close()
def __getitem__(self, key): def __getitem__(self, key):
key = self.key_filter_in(key) key = self.key_filter_in(key)
try: try:
@@ -357,12 +357,10 @@ class CacheOnDisk(CacheAbstract):
val_file.close() val_file.close()
return value return value
def __contains__(self, key): def __contains__(self, key):
key = self.key_filter_in(key) key = self.key_filter_in(key)
return (key in self.file_locks) or recfile.exists(key, path=self.folder) return (key in self.file_locks) or recfile.exists(key, path=self.folder)
def __delitem__(self, key): def __delitem__(self, key):
key = self.key_filter_in(key) key = self.key_filter_in(key)
try: try:
@@ -370,13 +368,11 @@ class CacheOnDisk(CacheAbstract):
except IOError: except IOError:
raise KeyError raise KeyError
def __iter__(self): def __iter__(self):
for dirpath, dirnames, filenames in os.walk(self.folder): for dirpath, dirnames, filenames in os.walk(self.folder):
for filename in filenames: for filename in filenames:
yield self.key_filter_out(filename) yield self.key_filter_out(filename)
def safe_apply(self, key, function, default_value=None): def safe_apply(self, key, function, default_value=None):
""" """
Safely apply a function to the value of a key in storage and set Safely apply a function to the value of a key in storage and set
@@ -403,25 +399,21 @@ class CacheOnDisk(CacheAbstract):
val_file.close() val_file.close()
return new_value return new_value
def keys(self): def keys(self):
return list(self.__iter__()) return list(self.__iter__())
def get(self, key, default=None): def get(self, key, default=None):
try: try:
return self[key] return self[key]
except KeyError: except KeyError:
return default return default
def __init__(self, request=None, folder=None): def __init__(self, request=None, folder=None):
self.initialized = False self.initialized = False
self.request = request self.request = request
self.folder = folder self.folder = folder
self.storage = None self.storage = None
def initialize(self): def initialize(self):
if self.initialized: if self.initialized:
return return
@@ -440,7 +432,6 @@ class CacheOnDisk(CacheAbstract):
self.storage = CacheOnDisk.PersistentStorage(folder) self.storage = CacheOnDisk.PersistentStorage(folder)
def __call__(self, key, f, def __call__(self, key, f,
time_expire=DEFAULT_TIME_EXPIRE): time_expire=DEFAULT_TIME_EXPIRE):
self.initialize() self.initialize()
@@ -487,7 +478,6 @@ class CacheOnDisk(CacheAbstract):
self.storage.release(key) self.storage.release(key)
return value return value
def clear(self, regex=None): def clear(self, regex=None):
self.initialize() self.initialize()
storage = self.storage storage = self.storage
@@ -504,7 +494,6 @@ class CacheOnDisk(CacheAbstract):
pass pass
storage.release(key) storage.release(key)
def increment(self, key, value=1): def increment(self, key, value=1):
self.initialize() self.initialize()
self.storage.acquire(key) self.storage.acquire(key)
@@ -513,7 +502,6 @@ class CacheOnDisk(CacheAbstract):
return value return value
class CacheAction(object): class CacheAction(object):
def __init__(self, func, key, time_expire, cache, cache_model): def __init__(self, func, key, time_expire, cache, cache_model):
self.__name__ = func.__name__ self.__name__ = func.__name__
@@ -572,9 +560,9 @@ class Cache(object):
logger.warning('no cache.disk (AttributeError)') logger.warning('no cache.disk (AttributeError)')
def action(self, time_expire=DEFAULT_TIME_EXPIRE, cache_model=None, def action(self, time_expire=DEFAULT_TIME_EXPIRE, cache_model=None,
prefix=None, session=False, vars=True, lang=True, prefix=None, session=False, vars=True, lang=True,
user_agent=False, public=True, valid_statuses=None, user_agent=False, public=True, valid_statuses=None,
quick=None): quick=None):
"""Better fit for caching an action """Better fit for caching an action
Warning: Warning:
@@ -602,6 +590,7 @@ class Cache(object):
""" """
from gluon import current from gluon import current
from gluon.http import HTTP from gluon.http import HTTP
def wrap(func): def wrap(func):
def wrapped_f(): def wrapped_f():
if current.request.env.request_method != 'GET': if current.request.env.request_method != 'GET':
@@ -621,13 +610,14 @@ class Cache(object):
cache_control = 'max-age=%(time_expire)s, s-maxage=%(time_expire)s' % dict(time_expire=time_expire) cache_control = 'max-age=%(time_expire)s, s-maxage=%(time_expire)s' % dict(time_expire=time_expire)
if not session_ and public_: if not session_ and public_:
cache_control += ', public' cache_control += ', public'
expires = (current.request.utcnow + datetime.timedelta(seconds=time_expire)).strftime('%a, %d %b %Y %H:%M:%S GMT') expires = (current.request.utcnow + datetime.timedelta(seconds=time_expire)
).strftime('%a, %d %b %Y %H:%M:%S GMT')
else: else:
cache_control += ', private' cache_control += ', private'
expires = 'Fri, 01 Jan 1990 00:00:00 GMT' expires = 'Fri, 01 Jan 1990 00:00:00 GMT'
if cache_model: if cache_model:
#figure out the correct cache key # figure out the correct cache key
cache_key = [current.request.env.path_info, current.response.view] cache_key = [current.request.env.path_info, current.response.view]
if session_: if session_:
cache_key.append(current.response.session_id) cache_key.append(current.response.session_id)
@@ -644,28 +634,28 @@ class Cache(object):
if prefix: if prefix:
cache_key = prefix + cache_key cache_key = prefix + cache_key
try: try:
#action returns something # action returns something
rtn = cache_model(cache_key, lambda : func(), time_expire=time_expire) rtn = cache_model(cache_key, lambda: func(), time_expire=time_expire)
http, status = None, current.response.status http, status = None, current.response.status
except HTTP, e: except HTTP, e:
#action raises HTTP (can still be valid) # action raises HTTP (can still be valid)
rtn = cache_model(cache_key, lambda : e.body, time_expire=time_expire) rtn = cache_model(cache_key, lambda: e.body, time_expire=time_expire)
http, status = HTTP(e.status, rtn, **e.headers), e.status http, status = HTTP(e.status, rtn, **e.headers), e.status
else: else:
#action raised a generic exception # action raised a generic exception
http = None http = None
else: else:
#no server-cache side involved # no server-cache side involved
try: try:
#action returns something # action returns something
rtn = func() rtn = func()
http, status = None, current.response.status http, status = None, current.response.status
except HTTP, e: except HTTP, e:
#action raises HTTP (can still be valid) # action raises HTTP (can still be valid)
status = e.status status = e.status
http = HTTP(e.status, e.body, **e.headers) http = HTTP(e.status, e.body, **e.headers)
else: else:
#action raised a generic exception # action raised a generic exception
http = None http = None
send_headers = False send_headers = False
if http and isinstance(valid_statuses, list): if http and isinstance(valid_statuses, list):
@@ -675,15 +665,13 @@ class Cache(object):
if str(status)[0] in '123': if str(status)[0] in '123':
send_headers = True send_headers = True
if send_headers: if send_headers:
headers = { headers = {'Pragma': None,
'Pragma' : None, 'Expires': expires,
'Expires' : expires, 'Cache-Control': cache_control}
'Cache-Control' : cache_control
}
current.response.headers.update(headers) current.response.headers.update(headers)
if cache_model and not send_headers: if cache_model and not send_headers:
#we cached already the value, but the status is not valid # we cached already the value, but the status is not valid
#so we need to delete the cached value # so we need to delete the cached value
cache_model(cache_key, None) cache_model(cache_key, None)
if http: if http:
if send_headers: if send_headers:
@@ -740,8 +728,7 @@ class Cache(object):
allow replacing cache.ram with cache.with_prefix(cache.ram,'prefix') allow replacing cache.ram with cache.with_prefix(cache.ram,'prefix')
it will add prefix to all the cache keys used. it will add prefix to all the cache keys used.
""" """
return lambda key, f, time_expire=DEFAULT_TIME_EXPIRE, prefix=prefix:\ return lambda key, f, time_expire=DEFAULT_TIME_EXPIRE, prefix=prefix: cache_model(prefix + key, f, time_expire)
cache_model(prefix + key, f, time_expire)
def lazy_cache(key=None, time_expire=None, cache_model='ram'): def lazy_cache(key=None, time_expire=None, cache_model='ram'):