Dropbox.web2py.gluon.cache

1 #!/usr/bin/env python 2 # -*- coding: utf-8 -*- 3 4 """ 5 This file is part of the web2py Web Framework 6 Copyrighted by Massimo Di Pierro <mdipierro@cs.depaul.edu> 7 License: LGPLv3 (http://www.gnu.org/licenses/lgpl.html) 8 9 Basic caching classes and methods 10 ================================= 11 12 - Cache - The generic caching object interfacing with the others 13 - CacheInRam - providing caching in ram 14 - CacheOnDisk - provides caches on disk 15 16 Memcache is also available via a different module (see gluon.contrib.memcache) 17 18 When web2py is running on Google App Engine, 19 caching will be provided by the GAE memcache 20 (see gluon.contrib.gae_memcache) 21 """ 22 import traceback 23 import time 24 import portalocker 25 import shelve 26 import thread 27 import os 28 import logging 29 import re 30 try: 31 import settings 32 have_settings = True 33 except ImportError: 34 have_settings = False 35 36 logger = logging.getLogger("web2py.cache") 37 38 __all__ = ['Cache', 'lazy_cache'] 39 40 41 DEFAULT_TIME_EXPIRE = 300

42 43 44 -class CacheAbstract(object):

45 """ 46 Abstract class for cache implementations. 47 Main function is now to provide referenced api documentation. 48 49 Use CacheInRam or CacheOnDisk instead which are derived from this class. 50 51 Attentions, Michele says: 52 53 There are signatures inside gdbm files that are used directly 54 by the python gdbm adapter that often are lagging behind in the 55 detection code in python part. 56 On every occasion that a gdbm store is probed by the python adapter, 57 the probe fails, because gdbm file version is newer. 58 Using gdbm directly from C would work, because there is backward 59 compatibility, but not from python! 60 The .shelve file is discarded and a new one created (with new 61 signature) and it works until it is probed again... 62 The possible consequences are memory leaks and broken sessions. 63 """ 64 65 cache_stats_name = 'web2py_cache_statistics' 66

67 - def __init__(self, request=None):

68 """ 69 Paremeters 70 ---------- 71 request: 72 the global request object 73 """ 74 raise NotImplementedError

75

76 - def __call__(self, key, f, 77 time_expire=DEFAULT_TIME_EXPIRE):

78 """ 79 Tries retrieve the value corresponding to `key` from the cache of the 80 object exists and if it did not expire, else it called the function `f` 81 and stores the output in the cache corresponding to `key`. In the case 82 the output of the function is returned. 83 84 :param key: the key of the object to be store or retrieved 85 :param f: the function, whose output is to be cached 86 :param time_expire: expiration of the cache in microseconds 87 88 - `time_expire` is used to compare the current time with the time when 89 the requested object was last saved in cache. It does not affect 90 future requests. 91 - Setting `time_expire` to 0 or negative value forces the cache to 92 refresh. 93 94 If the function `f` is `None` the cache is cleared. 95 """ 96 raise NotImplementedError

97

98 - def clear(self, regex=None):

99 """ 100 Clears the cache of all keys that match the provided regular expression. 101 If no regular expression is provided, it clears all entries in cache. 102 103 Parameters 104 ---------- 105 regex: 106 if provided, only keys matching the regex will be cleared. 107 Otherwise all keys are cleared. 108 """ 109 110 raise NotImplementedError

111

112 - def increment(self, key, value=1):

113 """ 114 Increments the cached value for the given key by the amount in value 115 116 Parameters 117 ---------- 118 key: 119 key for the cached object to be incremeneted 120 value: 121 amount of the increment (defaults to 1, can be negative) 122 """ 123 raise NotImplementedError

124

125 - def _clear(self, storage, regex):

126 """ 127 Auxiliary function called by `clear` to search and clear cache entries 128 """ 129 r = re.compile(regex) 130 for (key, value) in storage.items(): 131 if r.match(str(key)): 132 del storage[key]

133

134 135 -class CacheInRam(CacheAbstract):

136 """ 137 Ram based caching 138 139 This is implemented as global (per process, shared by all threads) 140 dictionary. 141 A mutex-lock mechanism avoid conflicts. 142 """ 143 144 locker = thread.allocate_lock() 145 meta_storage = {} 146

147 - def __init__(self, request=None):

148 self.initialized = False 149 self.request = request 150 self.storage = {}

151

152 - def initialize(self):

153 if self.initialized: 154 return 155 else: 156 self.initialized = True 157 self.locker.acquire() 158 request = self.request 159 if request: 160 app = request.application 161 else: 162 app = '' 163 if not app in self.meta_storage: 164 self.storage = self.meta_storage[app] = { 165 CacheAbstract.cache_stats_name: {'hit_total': 0, 'misses': 0}} 166 else: 167 self.storage = self.meta_storage[app] 168 self.locker.release()

169

170 - def clear(self, regex=None):

171 self.initialize() 172 self.locker.acquire() 173 storage = self.storage 174 if regex is None: 175 storage.clear() 176 else: 177 self._clear(storage, regex) 178 179 if not CacheAbstract.cache_stats_name in storage.keys(): 180 storage[CacheAbstract.cache_stats_name] = { 181 'hit_total': 0, 'misses': 0} 182 183 self.locker.release()

184

185 - def __call__(self, key, f, 186 time_expire=DEFAULT_TIME_EXPIRE, 187 destroyer=None):

188 """ 189 Attention! cache.ram does not copy the cached object. It just stores a reference to it. 190 Turns out the deepcopying the object has some problems: 191 1) would break backward compatibility 192 2) would be limiting because people may want to cache live objects 193 3) would work unless we deepcopy no storage and retrival which would make things slow. 194 Anyway. You can deepcopy explicitly in the function generating the value to be cached. 195 """ 196 self.initialize() 197 198 dt = time_expire 199 now = time.time() 200 201 self.locker.acquire() 202 item = self.storage.get(key, None) 203 if item and f is None: 204 del self.storage[key] 205 if destroyer: 206 destroyer(item[1]) 207 self.storage[CacheAbstract.cache_stats_name]['hit_total'] += 1 208 self.locker.release() 209 210 if f is None: 211 return None 212 if item and (dt is None or item[0] > now - dt): 213 return item[1] 214 elif item and (item[0] < now - dt) and destroyer: 215 destroyer(item[1]) 216 value = f() 217 218 self.locker.acquire() 219 self.storage[key] = (now, value) 220 self.storage[CacheAbstract.cache_stats_name]['misses'] += 1 221 self.locker.release() 222 return value

223

224 - def increment(self, key, value=1):

225 self.initialize() 226 self.locker.acquire() 227 try: 228 if key in self.storage: 229 value = self.storage[key][1] + value 230 self.storage[key] = (time.time(), value) 231 except BaseException, e: 232 self.locker.release() 233 raise e 234 self.locker.release() 235 return value

236

237 238 -class CacheOnDisk(CacheAbstract):

239 """ 240 Disk based cache 241 242 This is implemented as a shelve object and it is shared by multiple web2py 243 processes (and threads) as long as they share the same filesystem. 244 The file is locked wen accessed. 245 246 Disk cache provides persistance when web2py is started/stopped but it slower 247 than `CacheInRam` 248 249 Values stored in disk cache must be pickable. 250 """ 251

252 - def _close_shelve_and_unlock(self):

253 try: 254 if self.storage: 255 self.storage.close() 256 finally: 257 if self.locker and self.locked: 258 portalocker.unlock(self.locker) 259 self.locker.close() 260 self.locked = False

261

262 - def _open_shelve_and_lock(self):

263 """Open and return a shelf object, obtaining an exclusive lock 264 on self.locker first. Replaces the close method of the 265 returned shelf instance with one that releases the lock upon 266 closing.""" 267 268 storage = None 269 locker = None 270 locked = False 271 try: 272 locker = locker = open(self.locker_name, 'a') 273 portalocker.lock(locker, portalocker.LOCK_EX) 274 locked = True 275 try: 276 storage = shelve.open(self.shelve_name) 277 except: 278 logger.error('corrupted cache file %s, will try rebuild it' 279 % (self.shelve_name)) 280 storage = None 281 if not storage and os.path.exists(self.shelve_name): 282 os.unlink(self.shelve_name) 283 storage = shelve.open(self.shelve_name) 284 if not CacheAbstract.cache_stats_name in storage.keys(): 285 storage[CacheAbstract.cache_stats_name] = { 286 'hit_total': 0, 'misses': 0} 287 storage.sync() 288 except Exception, e: 289 if storage: 290 storage.close() 291 storage = None 292 if locked: 293 portalocker.unlock(locker) 294 locker.close() 295 locked = False 296 raise RuntimeError( 297 'unable to create/re-create cache file %s' % self.shelve_name) 298 self.locker = locker 299 self.locked = locked 300 self.storage = storage 301 return storage

302

303 - def __init__(self, request=None, folder=None):

304 self.initialized = False 305 self.request = request 306 self.folder = folder 307 self.storage = {}

308

309 - def initialize(self):

310 if self.initialized: 311 return 312 else: 313 self.initialized = True 314 folder = self.folder 315 request = self.request 316 317 # Lets test if the cache folder exists, if not 318 # we are going to create it 319 folder = folder or os.path.join(request.folder, 'cache') 320 321 if not os.path.exists(folder): 322 os.mkdir(folder) 323 324 ### we need this because of a possible bug in shelve that may 325 ### or may not lock 326 self.locker_name = os.path.join(folder, 'cache.lock') 327 self.shelve_name = os.path.join(folder, 'cache.shelve')

328

329 - def clear(self, regex=None):

330 self.initialize() 331 storage = self._open_shelve_and_lock() 332 try: 333 if regex is None: 334 storage.clear() 335 else: 336 self._clear(storage, regex) 337 storage.sync() 338 finally: 339 self._close_shelve_and_unlock()

340

341 - def __call__(self, key, f, 342 time_expire=DEFAULT_TIME_EXPIRE):

343 self.initialize() 344 dt = time_expire 345 storage = self._open_shelve_and_lock() 346 try: 347 item = storage.get(key, None) 348 storage[CacheAbstract.cache_stats_name]['hit_total'] += 1 349 if item and f is None: 350 del storage[key] 351 storage.sync() 352 now = time.time() 353 if f is None: 354 value = None 355 elif item and (dt is None or item[0] > now - dt): 356 value = item[1] 357 else: 358 value = f() 359 storage[key] = (now, value) 360 storage[CacheAbstract.cache_stats_name]['misses'] += 1 361 storage.sync() 362 finally: 363 self._close_shelve_and_unlock() 364 365 return value

366

367 - def increment(self, key, value=1):

368 self.initialize() 369 storage = self._open_shelve_and_lock() 370 try: 371 if key in storage: 372 value = storage[key][1] + value 373 storage[key] = (time.time(), value) 374 storage.sync() 375 finally: 376 self._close_shelve_and_unlock() 377 return value

378

379 380 -class CacheAction(object):

381 - def __init__(self, func, key, time_expire, cache, cache_model):

382 self.__name__ = func.__name__ 383 self.__doc__ = func.__doc__ 384 self.func = func 385 self.key = key 386 self.time_expire = time_expire 387 self.cache = cache 388 self.cache_model = cache_model

389

390 - def __call__(self, *a, **b):

391 if not self.key: 392 key2 = self.__name__ + ':' + repr(a) + ':' + repr(b) 393 else: 394 key2 = self.key.replace('%(name)s', self.__name__)\ 395 .replace('%(args)s', str(a)).replace('%(vars)s', str(b)) 396 cache_model = self.cache_model 397 if not cache_model or isinstance(cache_model, str): 398 cache_model = getattr(self.cache, cache_model or 'ram') 399 return cache_model(key2, 400 lambda a=a, b=b: self.func(*a, **b), 401 self.time_expire)

402

403 404 -class Cache(object):

405 """ 406 Sets up generic caching, creating an instance of both CacheInRam and 407 CacheOnDisk. 408 In case of GAE will make use of gluon.contrib.gae_memcache. 409 410 - self.ram is an instance of CacheInRam 411 - self.disk is an instance of CacheOnDisk 412 """ 413 414 autokey = ':%(name)s:%(args)s:%(vars)s' 415

416 - def __init__(self, request):

417 """ 418 Parameters 419 ---------- 420 request: 421 the global request object 422 """ 423 # GAE will have a special caching 424 if have_settings and settings.global_settings.web2py_runtime_gae: 425 from contrib.gae_memcache import MemcacheClient 426 self.ram = self.disk = MemcacheClient(request) 427 else: 428 # Otherwise use ram (and try also disk) 429 self.ram = CacheInRam(request) 430 try: 431 self.disk = CacheOnDisk(request) 432 except IOError: 433 logger.warning('no cache.disk (IOError)') 434 except AttributeError: 435 # normally not expected anymore, as GAE has already 436 # been accounted for 437 logger.warning('no cache.disk (AttributeError)')

438

439 - def __call__(self, 440 key=None, 441 time_expire=DEFAULT_TIME_EXPIRE, 442 cache_model=None):

443 """ 444 Decorator function that can be used to cache any function/method. 445 446 Example:: 447 448 @cache('key', 5000, cache.ram) 449 def f(): 450 return time.ctime() 451 452 When the function f is called, web2py tries to retrieve 453 the value corresponding to `key` from the cache of the 454 object exists and if it did not expire, else it calles the function `f` 455 and stores the output in the cache corresponding to `key`. In the case 456 the output of the function is returned. 457 458 :param key: the key of the object to be store or retrieved 459 :param time_expire: expiration of the cache in microseconds 460 :param cache_model: "ram", "disk", or other 461 (like "memcache" if defined). It defaults to "ram". 462 463 Notes 464 ----- 465 `time_expire` is used to compare the curret time with the time when the 466 requested object was last saved in cache. It does not affect future 467 requests. 468 Setting `time_expire` to 0 or negative value forces the cache to 469 refresh. 470 471 If the function `f` is an action, we suggest using 472 `request.env.path_info` as key. 473 """ 474 475 def tmp(func, cache=self, cache_model=cache_model): 476 return CacheAction(func, key, time_expire, self, cache_model)

477 return tmp

478 479 @staticmethod

480 - def with_prefix(cache_model, prefix):

481 """ 482 allow replacing cache.ram with cache.with_prefix(cache.ram,'prefix') 483 it will add prefix to all the cache keys used. 484 """ 485 return lambda key, f, time_expire=DEFAULT_TIME_EXPIRE, prefix=prefix:\ 486 cache_model(prefix + key, f, time_expire)

487

488 489 -def lazy_cache(key=None, time_expire=None, cache_model='ram'):

490 """ 491 can be used to cache any function including in modules, 492 as long as the cached function is only called within a web2py request 493 if a key is not provided, one is generated from the function name 494 the time_expire defaults to None (no cache expiration) 495 if cache_model is "ram" then the model is current.cache.ram, etc. 496 """ 497 def decorator(f, key=key, time_expire=time_expire, cache_model=cache_model): 498 key = key or repr(f) 499 500 def g(*c, **d): 501 from gluon import current 502 return current.cache(key, time_expire, cache_model)(f)(*c, **d)

503 g.__name__ = f.__name__ 504 return g 505 return decorator 506

Source Code for Module Dropbox.web2py.gluon.cache