dinislage/web2py
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

many improvements in cache

Browse Source
This commit is contained in:
mdipierro
2012-10-10 11:01:09 -05:00
parent 59ab818f5b
commit bf6f090428
4 changed files with 94 additions and 122 deletions
Download Patch File Download Diff File Expand all files Collapse all files
VERSION
+1 -1
View File
@@ -1 +1 @@
Version 2.1.0 (2012-10-10 09:18:57) dev Version 2.1.0 (2012-10-10 11:01:05) dev
gluon/cache.py
+83 -112
View File
@@ -11,7 +11,7 @@ Basic caching classes and methods
- Cache - The generic caching object interfacing with the others - Cache - The generic caching object interfacing with the others
- CacheInRam - providing caching in ram - CacheInRam - providing caching in ram
- CacheInDisk - provides caches on disk - CacheOnDisk - provides caches on disk
Memcache is also available via a different module (see gluon.contrib.memcache) Memcache is also available via a different module (see gluon.contrib.memcache)
@@ -130,22 +130,27 @@ class CacheInRam(CacheAbstract):
meta_storage = {} meta_storage = {}
def __init__(self, request=None): def __init__(self, request=None):
self.locker.acquire() self.initialized = False
self.request = request self.request = request
def initialize(self):
if self.initialized: return
else: self.initialized = True
self.locker.acquire()
request = self.request
if request: if request:
app = request.application app = request.application
else: else:
app = '' app = ''
if not app in self.meta_storage: if not app in self.meta_storage:
self.storage = self.meta_storage[app] = {CacheAbstract.cache_stats_name: { self.storage = self.meta_storage[app] = {
'hit_total': 0, CacheAbstract.cache_stats_name: {'hit_total': 0, 'misses': 0}}
'misses': 0,
}}
else: else:
self.storage = self.meta_storage[app] self.storage = self.meta_storage[app]
self.locker.release() self.locker.release()
def clear(self, regex=None): def clear(self, regex=None):
self.initialize()
self.locker.acquire() self.locker.acquire()
storage = self.storage storage = self.storage
if regex is None: if regex is None:
@@ -154,10 +159,7 @@ class CacheInRam(CacheAbstract):
self._clear(storage, regex) self._clear(storage, regex)
if not CacheAbstract.cache_stats_name in storage.keys(): if not CacheAbstract.cache_stats_name in storage.keys():
storage[CacheAbstract.cache_stats_name] = { storage[CacheAbstract.cache_stats_name] = {'hit_total': 0,'misses': 0}
'hit_total': 0,
'misses': 0,
}
self.locker.release() self.locker.release()
@@ -172,6 +174,7 @@ class CacheInRam(CacheAbstract):
3) would work unless we deepcopy no storage and retrival which would make things slow. 3) would work unless we deepcopy no storage and retrival which would make things slow.
Anyway. You can deepcopy explicitly in the function generating the value to be cached. Anyway. You can deepcopy explicitly in the function generating the value to be cached.
""" """
self.initialize()
dt = time_expire dt = time_expire
now = time.time() now = time.time()
@@ -200,6 +203,7 @@ class CacheInRam(CacheAbstract):
return value return value
def increment(self, key, value=1): def increment(self, key, value=1):
self.initialize()
self.locker.acquire() self.locker.acquire()
try: try:
if key in self.storage: if key in self.storage:
@@ -226,58 +230,66 @@ class CacheOnDisk(CacheAbstract):
Values stored in disk cache must be pickable. Values stored in disk cache must be pickable.
""" """
speedup_checks = set() def _close_shelve_and_unlock(self):
try:
if self.storage:
self.storage.close()
finally:
if self.locker and self.locked:
portalocker.unlock(self.locker)
self.locker.close()
self.locked = False
def _open_shelf_with_lock(self): def _open_shelve_and_lock(self):
"""Open and return a shelf object, obtaining an exclusive lock """Open and return a shelf object, obtaining an exclusive lock
on self.locker first. Replaces the close method of the on self.locker first. Replaces the close method of the
returned shelf instance with one that releases the lock upon returned shelf instance with one that releases the lock upon
closing.""" closing."""
def _close(self):
try: storage = None
shelve.Shelf.close(self) locker = None
finally: locked = False
portalocker.unlock(self.locker)
self.locker.close()
storage, locker, locker_locked = None, None, False
try: try:
locker = open(self.locker_name, 'a') locker = locker = open(self.locker_name, 'a')
portalocker.lock(locker, portalocker.LOCK_EX) portalocker.lock(locker, portalocker.LOCK_EX)
locker_locked = True locked = True
storage = shelve.open(self.shelve_name) try:
storage.close = _close.__get__(storage, shelve.Shelf) storage = shelve.open(self.shelve_name)
storage.locker = locker except:
except Exception: logger.error('corrupted cache file %s, will try rebuild it' \
logger.error('corrupted cache file %s, will try to delete and recreate it!' % (self.shelve_name)) % (self.shelve_name))
storage = None
if not storage and os.path.exists(self.shelve_name):
os.unlink(self.shelve_name)
storage = shelve.open(self.shelve_name)
if not CacheAbstract.cache_stats_name in storage.keys():
storage[CacheAbstract.cache_stats_name] = {'hit_total':0, 'misses': 0}
storage.sync()
except Exception, e:
if storage: if storage:
storage.close() storage.close()
storage = None storage = None
if locked:
try: portalocker.unlock(locker)
os.unlink(self.shelve_name) locker.close()
storage = shelve.open(self.shelve_name) locked = False
storage.close = _close.__get__(storage, shelve.Shelf) raise RuntimeError, 'unable to create/re-create cache file %s' % self.shelve_name
storage.locker = locker self.locker = locker
if not CacheAbstract.cache_stats_name in storage.keys(): self.locked = locked
storage[CacheAbstract.cache_stats_name] = { self.storage = storage
'hit_total': 0,
'misses': 0,
}
storage.sync()
except (IOError, OSError):
logger.warn('unable to delete and recreate cache file %s' % self.shelve_name)
if storage:
storage.close()
storage = None
if locker_locked:
portalocker.unlock(locker)
if locker:
locker.close()
return storage return storage
def __init__(self, request, folder=None): def __init__(self, request=None, folder=None):
self.initialized = False
self.request = request self.request = request
self.folder = folder
self.initialize()
def initialize(self):
if self.initialized: return
else: self.initialized = True
folder = self.folder
request = self.request
# Lets test if the cache folder exists, if not # Lets test if the cache folder exists, if not
# we are going to create it # we are going to create it
@@ -291,93 +303,54 @@ class CacheOnDisk(CacheAbstract):
self.locker_name = os.path.join(folder,'cache.lock') self.locker_name = os.path.join(folder,'cache.lock')
self.shelve_name = os.path.join(folder,'cache.shelve') self.shelve_name = os.path.join(folder,'cache.shelve')
speedup_key = (folder,CacheAbstract.cache_stats_name)
if not speedup_key in self.speedup_checks or \
not os.path.exists(self.shelve_name):
try:
storage = self._open_shelf_with_lock()
try:
if not CacheAbstract.cache_stats_name in storage:
storage[CacheAbstract.cache_stats_name] = {
'hit_total': 0,
'misses': 0,
}
storage.sync()
finally:
storage.close()
self.speedup_checks.add(speedup_key)
except ImportError:
pass # no module _bsddb, ignoring exception now so it makes a ticket only if used
def clear(self, regex=None): def clear(self, regex=None):
storage = self._open_shelf_with_lock() self.initialize()
storage = self._open_shelve_and_lock()
try: try:
if regex is None: if regex is None:
storage.clear() storage.clear()
else: else:
self._clear(storage, regex) self._clear(storage, regex)
if not CacheAbstract.cache_stats_name in storage.keys():
storage[CacheAbstract.cache_stats_name] = {
'hit_total': 0,
'misses': 0,
}
storage.sync() storage.sync()
finally: finally:
storage.close() self._close_shelve_and_unlock()
def __call__(self, key, f, def __call__(self, key, f,
time_expire = DEFAULT_TIME_EXPIRE): time_expire = DEFAULT_TIME_EXPIRE):
self.initialize()
dt = time_expire dt = time_expire
storage = self._open_shelve_and_lock()
storage = self._open_shelf_with_lock()
try: try:
item = storage.get(key, None) item = storage.get(key, None)
storage[CacheAbstract.cache_stats_name]['hit_total'] += 1
if item and f is None: if item and f is None:
del storage[key] del storage[key]
storage.sync()
storage[CacheAbstract.cache_stats_name] = { now = time.time()
'hit_total': storage[CacheAbstract.cache_stats_name]['hit_total'] + 1, if f is None:
'misses': storage[CacheAbstract.cache_stats_name]['misses'] value = None
} elif item and (dt is None or item[0] > now - dt):
value = item[1]
storage.sync() else:
value = f()
storage[key] = (now, value)
storage[CacheAbstract.cache_stats_name]['misses']+=1
storage.sync()
finally: finally:
if storage: self._close_shelve_and_unlock()
storage.close()
now = time.time()
if f is None:
return None
if item and (dt is None or item[0] > now - dt):
return item[1]
value = f()
storage = self._open_shelf_with_lock()
try:
storage[key] = (now, value)
storage[CacheAbstract.cache_stats_name] = {
'hit_total': storage[CacheAbstract.cache_stats_name]['hit_total'],
'misses': storage[CacheAbstract.cache_stats_name]['misses'] + 1
}
storage.sync()
finally:
if storage:
storage.close()
return value return value
def increment(self, key, value=1): def increment(self, key, value=1):
storage = self._open_shelf_with_lock() self.initialize()
storage = self._open_shelve_and_lock()
try: try:
if key in storage: if key in storage:
value = storage[key][1] + value value = storage[key][1] + value
storage[key] = (time.time(), value) storage[key] = (time.time(), value)
storage.sync() storage.sync()
finally: finally:
if storage: self._close_shelve_and_unlock()
storage.close()
return value return value
class CacheAction(object): class CacheAction(object):
@@ -423,10 +396,9 @@ class Cache(object):
the global request object the global request object
""" """
# GAE will have a special caching # GAE will have a special caching
if have_settings and settings.global_settings.web2py_runtime_gae: if have_settings and settings.global_settings.web2py_runtime_gae:
from contrib.gae_memcache import MemcacheClient from contrib.gae_memcache import MemcacheClient
self.ram=self.disk=MemcacheClient(request) self.ram = self.disk = MemcacheClient(request)
else: else:
# Otherwise use ram (and try also disk) # Otherwise use ram (and try also disk)
self.ram = CacheInRam(request) self.ram = CacheInRam(request)
@@ -489,7 +461,6 @@ class Cache(object):
cache_model(prefix + key, f, time_expire) cache_model(prefix + key, f, time_expire)
def lazy_cache(key=None,time_expire=None,cache_model='ram'): def lazy_cache(key=None,time_expire=None,cache_model='ram'):
""" """
can be used to cache any function including in modules, can be used to cache any function including in modules,
gluon/compileapp.py
+1
View File
@@ -387,6 +387,7 @@ def build_environment(request, response, session, store_current=True):
t = environment['T'] = translator(request) t = environment['T'] = translator(request)
c = environment['cache'] = Cache(request) c = environment['cache'] = Cache(request)
if store_current: if store_current:
current.globalenv = environment current.globalenv = environment
current.request = request current.request = request
gluon/main.py
+9 -9
View File
@@ -88,7 +88,7 @@ from contenttype import contenttype
from dal import BaseAdapter from dal import BaseAdapter
from settings import global_settings from settings import global_settings
from validators import CRYPT from validators import CRYPT
from cache import Cache from cache import CacheInRam
from html import URL, xmlescape from html import URL, xmlescape
from utils import is_valid_ip_address from utils import is_valid_ip_address
from rewrite import load, url_in, thread as rwthread, \ from rewrite import load, url_in, thread as rwthread, \
@@ -148,7 +148,7 @@ def get_client(env):
def copystream_progress(request, chunk_size= 10**5): def copystream_progress(request, chunk_size= 10**5):
""" """
copies request.env.wsgi_input into request.body copies request.env.wsgi_input into request.body
and stores progress upload status in cache.ram and stores progress upload status in cache_ram
X-Progress-ID:length and X-Progress-ID:uploaded X-Progress-ID:length and X-Progress-ID:uploaded
""" """
env = request.env env = request.env
@@ -164,16 +164,16 @@ def copystream_progress(request, chunk_size= 10**5):
copystream(source, dest, size, chunk_size) copystream(source, dest, size, chunk_size)
return dest return dest
cache_key = 'X-Progress-ID:'+request.vars['X-Progress-ID'] cache_key = 'X-Progress-ID:'+request.vars['X-Progress-ID']
cache = Cache(request) cache_ram = CacheInRam(request) # same as cache.ram because meta_storage
cache.ram(cache_key+':length', lambda: size, 0) cache_ram(cache_key+':length', lambda: size, 0)
cache.ram(cache_key+':uploaded', lambda: 0, 0) cache_ram(cache_key+':uploaded', lambda: 0, 0)
while size > 0: while size > 0:
if size < chunk_size: if size < chunk_size:
data = source.read(size) data = source.read(size)
cache.ram.increment(cache_key+':uploaded', size) cache_ram.increment(cache_key+':uploaded', size)
else: else:
data = source.read(chunk_size) data = source.read(chunk_size)
cache.ram.increment(cache_key+':uploaded', chunk_size) cache_ram.increment(cache_key+':uploaded', chunk_size)
length = len(data) length = len(data)
if length > size: if length > size:
(data, length) = (data[:size], size) (data, length) = (data[:size], size)
@@ -184,8 +184,8 @@ def copystream_progress(request, chunk_size= 10**5):
if length < chunk_size: if length < chunk_size:
break break
dest.seek(0) dest.seek(0)
cache.ram(cache_key+':length', None) cache_ram(cache_key+':length', None)
cache.ram(cache_key+':uploaded', None) cache_ram(cache_key+':uploaded', None)
return dest return dest