dinislage/web2py
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fixed locking issue with cache disk, thanks gypsy

Browse Source
This commit is contained in:
Massimo Di Pierro
2012-05-02 15:47:07 -05:00
parent 5d08e2a821
commit 6ddfbc883e
3 changed files with 81 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files
VERSION
+1 -1
View File
@@ -1 +1 @@
Version 1.99.7 (2012-05-02 14:27:33) dev
Version 1.99.7 (2012-05-02 15:44:58) dev
gluon/cache.py
+79 -58
View File
@@ -19,7 +19,7 @@ When web2py is running on Google App Engine,
caching will be provided by the GAE memcache
(see gluon.contrib.gae_memcache)
"""
import traceback
import time
import portalocker
import shelve
@@ -228,6 +228,54 @@ class CacheOnDisk(CacheAbstract):
speedup_checks = set()
def _open_shelf_with_lock(self):
"""Open and return a shelf object, obtaining an exclusive lock
on self.locker first. Replaces the close method of the
returned shelf instance with one that releases the lock upon
closing."""
def _close(self):
try:
shelve.Shelf.close(self)
finally:
portalocker.unlock(self.locker)
self.locker.close()
storage, locker, locker_locked = None, None, False
try:
locker = open(self.locker_name, 'a')
portalocker.lock(locker, portalocker.LOCK_EX)
locker_locked = True
storage = shelve.open(self.shelve_name)
storage.close = _close.__get__(storage, shelve.Shelf)
storage.locker = locker
except (Exception) as detail:
logger.error('corrupted cache file %s, will try to delete and recreate it!' % (self.shelve_name))
if storage:
storage.close()
storage = None
try:
os.unlink(self.shelve_name)
storage = shelve.open(self.shelve_name)
storage.close = _close.__get__(storage, shelve.Shelf)
storage.locker = locker
if not CacheAbstract.cache_stats_name in storage.keys():
storage[CacheAbstract.cache_stats_name] = {
'hit_total': 0,
'misses': 0,
}
storage.sync()
except (IOError, OSError):
logger.warn('unable to delete and recreate cache file %s' % self.shelve_name)
if storage:
storage.close()
storage = None
if locker_locked:
portalocker.unlock(locker)
if locker:
locker.close()
return storage
def __init__(self, request, folder=None):
self.request = request
@@ -243,45 +291,26 @@ class CacheOnDisk(CacheAbstract):
self.locker_name = os.path.join(folder,'cache.lock')
self.shelve_name = os.path.join(folder,'cache.shelve')
locker, locker_locked = None, False
speedup_key = (folder,CacheAbstract.cache_stats_name)
if not speedup_key in self.speedup_checks or \
not os.path.exists(self.shelve_name):
try:
locker = open(self.locker_name, 'a')
portalocker.lock(locker, portalocker.LOCK_EX)
locker_locked = True
storage = shelve.open(self.shelve_name)
storage = self._open_shelf_with_lock()
try:
if not storage.has_key(CacheAbstract.cache_stats_name):
storage[CacheAbstract.cache_stats_name] = {
'hit_total': 0,
'misses': 0,
}
}
storage.sync()
finally:
storage.close()
self.speedup_checks.add(speedup_key)
except ImportError:
pass # no module _bsddb, ignoring exception now so it makes a ticket only if used
except:
logger.error('corrupted file %s, will try delete it!' \
% self.shelve_name)
try:
os.unlink(self.shelve_name)
except IOError:
logger.warn('unable to delete file %s' % self.shelve_name)
except OSError:
logger.warn('unable to delete file %s' % self.shelve_name)
if locker_locked:
portalocker.unlock(locker)
if locker:
locker.close()
def clear(self, regex=None):
locker = open(self.locker_name,'a')
portalocker.lock(locker, portalocker.LOCK_EX)
storage = shelve.open(self.shelve_name)
storage = self._open_shelf_with_lock()
try:
if regex is None:
storage.clear()
@@ -295,30 +324,26 @@ class CacheOnDisk(CacheAbstract):
storage.sync()
finally:
storage.close()
portalocker.unlock(locker)
locker.close()
def __call__(self, key, f,
time_expire = DEFAULT_TIME_EXPIRE):
dt = time_expire
locker = open(self.locker_name,'a')
portalocker.lock(locker, portalocker.LOCK_EX)
storage = shelve.open(self.shelve_name)
storage = self._open_shelf_with_lock()
try:
item = storage.get(key, None)
if item and f is None:
del storage[key]
item = storage.get(key, None)
if item and f is None:
del storage[key]
storage[CacheAbstract.cache_stats_name] = {
'hit_total': storage[CacheAbstract.cache_stats_name]['hit_total'] + 1,
'misses': storage[CacheAbstract.cache_stats_name]['misses']
}
storage[CacheAbstract.cache_stats_name] = {
'hit_total': storage[CacheAbstract.cache_stats_name]['hit_total'] + 1,
'misses': storage[CacheAbstract.cache_stats_name]['misses']
}
storage.sync()
portalocker.unlock(locker)
locker.close()
storage.sync()
finally:
if storage:
storage.close()
if f is None:
return None
@@ -326,36 +351,32 @@ class CacheOnDisk(CacheAbstract):
return item[1]
value = f()
locker = open(self.locker_name,'a')
portalocker.lock(locker, portalocker.LOCK_EX)
storage[key] = (time.time(), value)
storage = self._open_shelf_with_lock()
try:
storage[key] = (time.time(), value)
storage[CacheAbstract.cache_stats_name] = {
'hit_total': storage[CacheAbstract.cache_stats_name]['hit_total'],
'misses': storage[CacheAbstract.cache_stats_name]['misses'] + 1
}
storage[CacheAbstract.cache_stats_name] = {
'hit_total': storage[CacheAbstract.cache_stats_name]['hit_total'],
'misses': storage[CacheAbstract.cache_stats_name]['misses'] + 1
}
storage.sync()
storage.close()
portalocker.unlock(locker)
locker.close()
storage.sync()
finally:
if storage:
storage.close()
return value
def increment(self, key, value=1):
locker = open(self.locker_name,'a')
portalocker.lock(locker, portalocker.LOCK_EX)
storage = shelve.open(self.shelve_name)
storage = self._open_shelf_with_lock()
try:
if key in storage:
value = storage[key][1] + value
storage[key] = (time.time(), value)
storage.sync()
finally:
storage.close()
portalocker.unlock(locker)
locker.close()
if storage:
storage.close()
return value
gluon/tools.py
+1
View File
@@ -1157,6 +1157,7 @@ class Auth(object):
def _get_user_id(self):
"accessor for auth.user_id"
return self.user and self.user.id or None
user_id = property(_get_user_id, doc="user.id or None")
def _HTTP(self, *a, **b):