dinislage/web2py
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

cache.py PEP8 enhancement

Browse Source
This commit is contained in:
Hardirc
2016-03-26 13:11:27 -04:00
parent 060aeff867
commit 58a8ba067c
1 changed files with 37 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files
gluon/cache.py
+37 -50
View File
@@ -44,9 +44,9 @@ except ImportError:
have_settings = False
try:
import cPickle as pickle
import cPickle as pickle
except:
import pickle
import pickle
try:
import psutil
@@ -54,6 +54,7 @@ try:
except ImportError:
HAVE_PSUTIL = False
def remove_oldest_entries(storage, percentage=90):
# compute current memory usage (%)
old_mem = psutil.virtual_memory().percent
@@ -66,7 +67,8 @@ def remove_oldest_entries(storage, percentage=90):
# comute used memory again
new_mem = psutil.virtual_memory().percent
# if the used memory did not decrease stop
if new_mem >= old_mem: break
if new_mem >= old_mem:
break
# net new measurement for memory usage and loop
old_mem = new_mem
@@ -78,6 +80,7 @@ __all__ = ['Cache', 'lazy_cache']
DEFAULT_TIME_EXPIRE = 300
class CacheAbstract(object):
"""
Abstract class for cache implementations.
@@ -99,7 +102,7 @@ class CacheAbstract(object):
"""
cache_stats_name = 'web2py_cache_statistics'
max_ram_utilization = None # percent
max_ram_utilization = None # percent
def __init__(self, request=None):
"""Initializes the object
@@ -182,13 +185,14 @@ class CacheInRam(CacheAbstract):
self.request = request
self.storage = OrderedDict() if HAVE_PSUTIL else {}
self.app = request.application if request else ''
def initialize(self):
if self.initialized:
return
else:
self.initialized = True
self.locker.acquire()
if not self.app in self.meta_storage:
if self.app not in self.meta_storage:
self.storage = self.meta_storage[self.app] = \
OrderedDict() if HAVE_PSUTIL else {}
self.stats[self.app] = {'hit_total': 0, 'misses': 0}
@@ -205,7 +209,7 @@ class CacheInRam(CacheAbstract):
else:
self._clear(storage, regex)
if not self.app in self.stats:
if self.app not in self.stats:
self.stats[self.app] = {'hit_total': 0, 'misses': 0}
self.locker.release()
@@ -251,8 +255,8 @@ class CacheInRam(CacheAbstract):
self.locker.acquire()
self.storage[key] = (now, value)
self.stats[self.app]['misses'] += 1
if HAVE_PSUTIL and self.max_ram_utilization!=None and random.random()<0.10:
remove_oldest_entries(self.storage, percentage = self.max_ram_utilization)
if HAVE_PSUTIL and self.max_ram_utilization is not None and random.random() < 0.10:
remove_oldest_entries(self.storage, percentage=self.max_ram_utilization)
self.locker.release()
return value
@@ -292,14 +296,15 @@ class CacheOnDisk(CacheAbstract):
self.folder = folder
self.key_filter_in = lambda key: key
self.key_filter_out = lambda key: key
self.file_lock_time_wait = file_lock_time_wait # How long we should wait before retrying to lock a file held by another process
self.file_lock_time_wait = file_lock_time_wait
# How long we should wait before retrying to lock a file held by another process
# We still need a mutex for each file as portalocker only blocks other processes
self.file_locks = defaultdict(thread.allocate_lock)
# Make sure we use valid filenames.
if sys.platform == "win32":
import base64
def key_filter_in_windows(key):
"""
Windows doesn't allow \ / : * ? "< > | in filenames.
@@ -316,7 +321,6 @@ class CacheOnDisk(CacheAbstract):
self.key_filter_in = key_filter_in_windows
self.key_filter_out = key_filter_out_windows
def wait_portalock(self, val_file):
"""
Wait for the process file lock.
@@ -328,15 +332,12 @@ class CacheOnDisk(CacheAbstract):
except:
time.sleep(self.file_lock_time_wait)
def acquire(self, key):
self.file_locks[key].acquire()
def release(self, key):
self.file_locks[key].release()
def __setitem__(self, key, value):
key = self.key_filter_in(key)
val_file = recfile.open(key, mode='wb', path=self.folder)
@@ -344,7 +345,6 @@ class CacheOnDisk(CacheAbstract):
pickle.dump(value, val_file, pickle.HIGHEST_PROTOCOL)
val_file.close()
def __getitem__(self, key):
key = self.key_filter_in(key)
try:
@@ -357,12 +357,10 @@ class CacheOnDisk(CacheAbstract):
val_file.close()
return value
def __contains__(self, key):
key = self.key_filter_in(key)
return (key in self.file_locks) or recfile.exists(key, path=self.folder)
def __delitem__(self, key):
key = self.key_filter_in(key)
try:
@@ -370,13 +368,11 @@ class CacheOnDisk(CacheAbstract):
except IOError:
raise KeyError
def __iter__(self):
for dirpath, dirnames, filenames in os.walk(self.folder):
for filename in filenames:
yield self.key_filter_out(filename)
def safe_apply(self, key, function, default_value=None):
"""
Safely apply a function to the value of a key in storage and set
@@ -403,25 +399,21 @@ class CacheOnDisk(CacheAbstract):
val_file.close()
return new_value
def keys(self):
return list(self.__iter__())
def get(self, key, default=None):
try:
return self[key]
except KeyError:
return default
def __init__(self, request=None, folder=None):
self.initialized = False
self.request = request
self.folder = folder
self.storage = None
def initialize(self):
if self.initialized:
return
@@ -440,7 +432,6 @@ class CacheOnDisk(CacheAbstract):
self.storage = CacheOnDisk.PersistentStorage(folder)
def __call__(self, key, f,
time_expire=DEFAULT_TIME_EXPIRE):
self.initialize()
@@ -487,7 +478,6 @@ class CacheOnDisk(CacheAbstract):
self.storage.release(key)
return value
def clear(self, regex=None):
self.initialize()
storage = self.storage
@@ -504,7 +494,6 @@ class CacheOnDisk(CacheAbstract):
pass
storage.release(key)
def increment(self, key, value=1):
self.initialize()
self.storage.acquire(key)
@@ -513,7 +502,6 @@ class CacheOnDisk(CacheAbstract):
return value
class CacheAction(object):
def __init__(self, func, key, time_expire, cache, cache_model):
self.__name__ = func.__name__
@@ -572,9 +560,9 @@ class Cache(object):
logger.warning('no cache.disk (AttributeError)')
def action(self, time_expire=DEFAULT_TIME_EXPIRE, cache_model=None,
prefix=None, session=False, vars=True, lang=True,
user_agent=False, public=True, valid_statuses=None,
quick=None):
prefix=None, session=False, vars=True, lang=True,
user_agent=False, public=True, valid_statuses=None,
quick=None):
"""Better fit for caching an action
Warning:
@@ -602,6 +590,7 @@ class Cache(object):
"""
from gluon import current
from gluon.http import HTTP
def wrap(func):
def wrapped_f():
if current.request.env.request_method != 'GET':
@@ -621,13 +610,14 @@ class Cache(object):
cache_control = 'max-age=%(time_expire)s, s-maxage=%(time_expire)s' % dict(time_expire=time_expire)
if not session_ and public_:
cache_control += ', public'
expires = (current.request.utcnow + datetime.timedelta(seconds=time_expire)).strftime('%a, %d %b %Y %H:%M:%S GMT')
expires = (current.request.utcnow + datetime.timedelta(seconds=time_expire)
).strftime('%a, %d %b %Y %H:%M:%S GMT')
else:
cache_control += ', private'
expires = 'Fri, 01 Jan 1990 00:00:00 GMT'
if cache_model:
#figure out the correct cache key
# figure out the correct cache key
cache_key = [current.request.env.path_info, current.response.view]
if session_:
cache_key.append(current.response.session_id)
@@ -644,28 +634,28 @@ class Cache(object):
if prefix:
cache_key = prefix + cache_key
try:
#action returns something
rtn = cache_model(cache_key, lambda : func(), time_expire=time_expire)
# action returns something
rtn = cache_model(cache_key, lambda: func(), time_expire=time_expire)
http, status = None, current.response.status
except HTTP, e:
#action raises HTTP (can still be valid)
rtn = cache_model(cache_key, lambda : e.body, time_expire=time_expire)
# action raises HTTP (can still be valid)
rtn = cache_model(cache_key, lambda: e.body, time_expire=time_expire)
http, status = HTTP(e.status, rtn, **e.headers), e.status
else:
#action raised a generic exception
# action raised a generic exception
http = None
else:
#no server-cache side involved
# no server-cache side involved
try:
#action returns something
# action returns something
rtn = func()
http, status = None, current.response.status
except HTTP, e:
#action raises HTTP (can still be valid)
# action raises HTTP (can still be valid)
status = e.status
http = HTTP(e.status, e.body, **e.headers)
else:
#action raised a generic exception
# action raised a generic exception
http = None
send_headers = False
if http and isinstance(valid_statuses, list):
@@ -675,15 +665,13 @@ class Cache(object):
if str(status)[0] in '123':
send_headers = True
if send_headers:
headers = {
'Pragma' : None,
'Expires' : expires,
'Cache-Control' : cache_control
}
headers = {'Pragma': None,
'Expires': expires,
'Cache-Control': cache_control}
current.response.headers.update(headers)
if cache_model and not send_headers:
#we cached already the value, but the status is not valid
#so we need to delete the cached value
# we cached already the value, but the status is not valid
# so we need to delete the cached value
cache_model(cache_key, None)
if http:
if send_headers:
@@ -740,8 +728,7 @@ class Cache(object):
allow replacing cache.ram with cache.with_prefix(cache.ram,'prefix')
it will add prefix to all the cache keys used.
"""
return lambda key, f, time_expire=DEFAULT_TIME_EXPIRE, prefix=prefix:\
cache_model(prefix + key, f, time_expire)
return lambda key, f, time_expire=DEFAULT_TIME_EXPIRE, prefix=prefix: cache_model(prefix + key, f, time_expire)
def lazy_cache(key=None, time_expire=None, cache_model='ram'):