fixed locking issue with cache disk, thanks gypsy

VERSION

@@ -1 +1 @@
-Version 1.99.7 (2012-05-02 14:27:33) dev
+Version 1.99.7 (2012-05-02 15:44:58) dev

gluon/cache.py

@@ -19,7 +19,7 @@ When web2py is running on Google App Engine,
 caching will be provided by the GAE memcache
 (see gluon.contrib.gae_memcache)
 """
-
+import traceback
 import time
 import portalocker
 import shelve
@@ -228,6 +228,54 @@ class CacheOnDisk(CacheAbstract):
 
     speedup_checks = set()
 
+    def _open_shelf_with_lock(self):
+        """Open and return a shelf object, obtaining an exclusive lock
+        on self.locker first. Replaces the close method of the
+        returned shelf instance with one that releases the lock upon
+        closing."""
+        def _close(self):
+            try:
+                shelve.Shelf.close(self)
+            finally:
+                portalocker.unlock(self.locker)
+                self.locker.close()
+
+        storage, locker, locker_locked = None, None, False
+        try:
+            locker = open(self.locker_name, 'a')
+            portalocker.lock(locker, portalocker.LOCK_EX)
+            locker_locked = True
+            storage = shelve.open(self.shelve_name)
+            storage.close = _close.__get__(storage, shelve.Shelf)
+            storage.locker = locker
+        except (Exception) as detail:
+            logger.error('corrupted cache file %s, will try to delete and recreate it!' % (self.shelve_name))
+            if storage:
+                storage.close()
+                storage = None
+
+            try:
+                os.unlink(self.shelve_name)
+                storage = shelve.open(self.shelve_name)
+                storage.close = _close.__get__(storage, shelve.Shelf)
+                storage.locker = locker
+                if not CacheAbstract.cache_stats_name in storage.keys():
+                    storage[CacheAbstract.cache_stats_name] = {
+                        'hit_total': 0,
+                        'misses': 0,
+                    }
+                storage.sync()
+            except (IOError, OSError):
+                logger.warn('unable to delete and recreate cache file %s' % self.shelve_name)
+                if storage:
+                        storage.close()
+                        storage = None
+                if locker_locked:
+                    portalocker.unlock(locker)
+                if locker:
+                    locker.close()
+        return storage
+
     def __init__(self, request, folder=None):
         self.request = request
 
@@ -243,45 +291,26 @@ class CacheOnDisk(CacheAbstract):
         self.locker_name = os.path.join(folder,'cache.lock')
         self.shelve_name = os.path.join(folder,'cache.shelve')
 
-        locker, locker_locked = None, False
         speedup_key = (folder,CacheAbstract.cache_stats_name)
         if not speedup_key in self.speedup_checks or \
                 not os.path.exists(self.shelve_name):
             try:
-                locker = open(self.locker_name, 'a')
+                storage = self._open_shelf_with_lock()
-                portalocker.lock(locker, portalocker.LOCK_EX)
-                locker_locked = True
-                storage = shelve.open(self.shelve_name)
                 try:
                     if not storage.has_key(CacheAbstract.cache_stats_name):
                         storage[CacheAbstract.cache_stats_name] = {
                             'hit_total': 0,
                             'misses': 0,
-                            }
+                        }
                         storage.sync()
                 finally:
                     storage.close()
                 self.speedup_checks.add(speedup_key)
             except ImportError:
                 pass # no module _bsddb, ignoring exception now so it makes a ticket only if used
-            except:
-                logger.error('corrupted file %s, will try delete it!' \
-                                 % self.shelve_name)
-                try:
-                    os.unlink(self.shelve_name)
-                except IOError:
-                    logger.warn('unable to delete file %s' % self.shelve_name)
-                except OSError:
-                    logger.warn('unable to delete file %s' % self.shelve_name)
-            if locker_locked:
-                portalocker.unlock(locker)
-            if locker:
-                locker.close()
 
     def clear(self, regex=None):
-        locker = open(self.locker_name,'a')
+        storage = self._open_shelf_with_lock()
-        portalocker.lock(locker, portalocker.LOCK_EX)
-        storage = shelve.open(self.shelve_name)
         try:
             if regex is None:
                 storage.clear()
@@ -295,30 +324,26 @@ class CacheOnDisk(CacheAbstract):
             storage.sync()
         finally:
             storage.close()
-        portalocker.unlock(locker)
-        locker.close()
 
     def __call__(self, key, f,
                 time_expire = DEFAULT_TIME_EXPIRE):
         dt = time_expire
 
-        locker = open(self.locker_name,'a')
+        storage = self._open_shelf_with_lock()
-        portalocker.lock(locker, portalocker.LOCK_EX)
+        try:
-        storage = shelve.open(self.shelve_name)
+            item = storage.get(key, None)
+            if item and f is None:
+                del storage[key]
 
-        item = storage.get(key, None)
+            storage[CacheAbstract.cache_stats_name] = {
-        if item and f is None:
+                'hit_total': storage[CacheAbstract.cache_stats_name]['hit_total'] + 1,
-            del storage[key]
+                'misses': storage[CacheAbstract.cache_stats_name]['misses']
+            }
 
-        storage[CacheAbstract.cache_stats_name] = {
+            storage.sync()
-            'hit_total': storage[CacheAbstract.cache_stats_name]['hit_total'] + 1,
+        finally:
-            'misses': storage[CacheAbstract.cache_stats_name]['misses']
+            if storage:
-        }
+                storage.close()
-
-        storage.sync()
-
-        portalocker.unlock(locker)
-        locker.close()
 
         if f is None:
             return None
@@ -326,36 +351,32 @@ class CacheOnDisk(CacheAbstract):
             return item[1]
         value = f()
 
-        locker = open(self.locker_name,'a')
+        storage = self._open_shelf_with_lock()
-        portalocker.lock(locker, portalocker.LOCK_EX)
+        try:
-        storage[key] = (time.time(), value)
+            storage[key] = (time.time(), value)
 
-        storage[CacheAbstract.cache_stats_name] = {
+            storage[CacheAbstract.cache_stats_name] = {
-            'hit_total': storage[CacheAbstract.cache_stats_name]['hit_total'],
+                'hit_total': storage[CacheAbstract.cache_stats_name]['hit_total'],
-            'misses': storage[CacheAbstract.cache_stats_name]['misses'] + 1
+                'misses': storage[CacheAbstract.cache_stats_name]['misses'] + 1
-        }
+            }
 
-        storage.sync()
+            storage.sync()
-
+        finally:
-        storage.close()
+            if storage:
-        portalocker.unlock(locker)
+                storage.close()
-        locker.close()
 
         return value
 
     def increment(self, key, value=1):
-        locker = open(self.locker_name,'a')
+        storage = self._open_shelf_with_lock()
-        portalocker.lock(locker, portalocker.LOCK_EX)
-        storage = shelve.open(self.shelve_name)
         try:
             if key in storage:
                 value = storage[key][1] + value
             storage[key] = (time.time(), value)
             storage.sync()
         finally:
-            storage.close()
+            if storage:
-            portalocker.unlock(locker)
+                storage.close()
-            locker.close()
         return value
 
 

gluon/tools.py

@@ -1157,6 +1157,7 @@ class Auth(object):
     def _get_user_id(self):
        "accessor for auth.user_id"
        return self.user and self.user.id or None
+
     user_id = property(_get_user_id, doc="user.id or None")
 
     def _HTTP(self, *a, **b):