dinislage/web2py
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

new redis toolset to use with web2py

Browse Source 
This is a refactor of everything web2py uses with redis.
Specifically:
- a refactored redis_cache.py that fixes #958, allowing a "fail
gracefully" behaviour in case redis is not available
- a refactored redis_session.py that blocks less with with_lock=True
(although still not optimal)
- a new (and NEEDED) redis_utils.py that serves as the base for
everything else, allowing an RConn object that you can freely use as a
redis.StrictRedis connection, and that you can override in case you're
using a different library (or that web2py will use in case redis-py
won't be the de-facto standard around)
- a newly - and much anticipated - redis_scheduler.py. It's a slip-in
replacement for the standard scheduler that uses redis for workers
coordination. Feel free to dig in the code and improve it.

For redis_cache and redis_session changes are BREAKING. It means that
users will need to change the import locations and tune a bit the code.
Now every module depends on an gluon.contrib.redis_utils.RConn object
(or similar) that in turns is very similar to a redis.StrictRedis one.
The redis instance is EXTERNAL to the modules themselves (no more "host,
port, db, password" parameters in RedisCache or RedisSession)
See the relevant docstrings for usage examples
This commit is contained in:
niphlod
2016-01-02 23:15:00 +01:00
parent 1ca0c9b0c0
commit 12acdb51d7
5 changed files with 980 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files
gluon/contrib/redis_cache.py
+52 -54
View File
@@ -2,20 +2,20 @@
Developed by niphlod@gmail.com
Released under web2py license because includes gluon/cache.py source code
"""
import redis
from redis.exceptions import ConnectionError
from gluon import current
from gluon.cache import CacheAbstract
try:
import cPickle as pickle
import cPickle as pickle
except:
import pickle
import pickle
import time
import re
import logging
import thread
import random
from gluon import current
from gluon.cache import CacheAbstract
from gluon.contrib.redis_utils import acquire_lock, release_lock
from gluon.contrib.redis_utils import register_release_lock, RConnectionError
logger = logging.getLogger("web2py.cache.redis")
@@ -24,17 +24,22 @@ locker = thread.allocate_lock()
def RedisCache(*args, **vars):
"""
Usage example: put in models
Usage example: put in models::
from gluon.contrib.redis_cache import RedisCache
cache.redis = RedisCache('localhost:6379',db=None, debug=True, with_lock=True, password=None)
from gluon.contrib.redis_utils import RConn
rconn = RConn()
from gluon.contrib.redis_cache import RedisCache
cache.redis = RedisCache(redis_conn=rconn, debug=True, with_lock=True)
:param db: redis db to use (0..16)
:param debug: if True adds to stats() the total_hits and misses
:param with_lock: sets the default locking mode for creating new keys.
Args:
redis_conn: a redis-like connection object
debug: if True adds to stats() the total_hits and misses
with_lock: sets the default locking mode for creating new keys.
By default is False (usualy when you choose Redis you do it
for performances reason)
When True, only one thread/process can set a value concurrently
fail_gracefully: if redis is unavailable, returns the value computing it
instead of raising an exception
When you use cache.redis directly you can use
value = cache.redis('mykey', lambda: time.time(), with_lock=True)
@@ -81,22 +86,19 @@ class RedisClient(object):
MAX_RETRIES = 5
RETRIES = 0
def __init__(self, server='localhost:6379', db=None, debug=False, with_lock=False, password=None):
self.server = server
self.password = password
self.db = db or 0
host, port = (self.server.split(':') + ['6379'])[:2]
port = int(port)
def __init__(self, redis_conn=None, debug=False,
with_lock=False, fail_gracefully=False):
self.request = current.request
self.debug = debug
self.with_lock = with_lock
self.prefix = "w2p:%s:" % (self.request.application)
self.fail_gracefully = fail_gracefully
self.prefix = "w2p:cache:%s:" % (self.request.application)
if self.request:
app = self.request.application
else:
app = ''
if not app in self.meta_storage:
if app not in self.meta_storage:
self.storage = self.meta_storage[app] = {
CacheAbstract.cache_stats_name: {
'hit_total': 0,
@@ -107,7 +109,8 @@ class RedisClient(object):
self.cache_set_key = 'w2p:%s:___cache_set' % (self.request.application)
self.r_server = redis.Redis(host=host, port=port, db=self.db, password=self.password)
self.r_server = redis_conn
self._release_script = register_release_lock(self.r_server)
def initialize(self):
pass
@@ -140,26 +143,20 @@ class RedisClient(object):
#naive distributed locking
if with_lock:
lock_key = '%s:__lock' % newKey
try:
while True:
lock = self.r_server.setnx(lock_key, 1)
if lock:
value = self.cache_it(newKey, f, time_expire)
break
else:
time.sleep(0.2)
#did someone else create it in the meanwhile ?
obj = self.r_server.get(newKey)
if obj:
value = pickle.loads(obj)
break
finally:
self.r_server.delete(lock_key)
randomvalue = time.time()
al = acquire_lock(self.r_server, lock_key, randomvalue)
#someone may have computed it
obj = self.r_server.get(newKey)
if obj is None:
value = self.cache_it(newKey, f, time_expire)
else:
value = pickle.loads(obj)
release_lock(self, lock_key, al)
else:
#without distributed locking
value = self.cache_it(newKey, f, time_expire)
return value
except ConnectionError:
except RConnectionError:
return self.retry_call(key, f, time_expire, with_lock)
def cache_it(self, key, f, time_expire):
@@ -172,39 +169,42 @@ class RedisClient(object):
value_ = pickle.dumps(value, pickle.HIGHEST_PROTOCOL)
if time_expire == 0:
time_expire = 1
self.r_server.setex(key, value_, time_expire)
self.r_server.setex(key, time_expire, value_)
#print '%s will expire on %s: it goes in bucket %s' % (key, time.ctime(expireat))
#print 'that will expire on %s' % (bucket_key, time.ctime(((expireat/60) + 1)*60))
p = self.r_server.pipeline()
#add bucket to the fixed set
p.sadd(cache_set_key, bucket_key)
#sets the key
p.setex(key, value_, time_expire)
p.setex(key, time_expire, value_)
#add the key to the bucket
p.sadd(bucket_key, key)
#expire the bucket properly
p.expireat(bucket_key, ((expireat/60) + 1)*60)
p.expireat(bucket_key, ((expireat / 60) + 1) * 60)
p.execute()
return value
def retry_call(self, key, f, time_expire, with_locking):
def retry_call(self, key, f, time_expire, with_lock):
self.RETRIES += 1
if self.RETRIES <= self.MAX_RETRIES:
logger.error("sleeping %s seconds before reconnecting" %
(2 * self.RETRIES))
time.sleep(2 * self.RETRIES)
self.__init__(self.server, self.db, self.debug, self.with_lock)
return self.__call__(key, f, time_expire, with_locking)
if self.fail_gracefully:
self.RETRIES = 0
return f()
return self.__call__(key, f, time_expire, with_lock)
else:
self.RETRIES = 0
raise ConnectionError('Redis instance is unavailable at %s' % (
self.server))
if self.fail_gracefully:
return f
raise RConnectionError('Redis instance is unavailable')
def increment(self, key, value=1):
try:
newKey = self.__keyFormat__(key)
return self.r_server.incr(newKey, value)
except ConnectionError:
except RConnectionError:
return self.retry_increment(key, value)
def retry_increment(self, key, value):
@@ -212,12 +212,10 @@ class RedisClient(object):
if self.RETRIES <= self.MAX_RETRIES:
logger.error("sleeping some seconds before reconnecting")
time.sleep(2 * self.RETRIES)
self.__init__(self.server, self.db, self.debug, self.with_lock)
return self.increment(key, value)
else:
self.RETRIES = 0
raise ConnectionError('Redis instance is unavailable at %s' % (
self.server))
raise RConnectionError('Redis instance is unavailable')
def clear(self, regex):
"""
@@ -225,9 +223,9 @@ class RedisClient(object):
clear cache entries
"""
r = re.compile(regex)
#get all buckets
# get all buckets
buckets = self.r_server.smembers(self.cache_set_key)
#get all keys in buckets
# get all keys in buckets
if buckets:
keys = self.r_server.sunion(buckets)
else:
@@ -237,8 +235,8 @@ class RedisClient(object):
for a in keys:
if r.match(str(a).replace(prefix, '', 1)):
pipe.delete(a)
if random.randrange(0,100) < 10:
#do this just once in a while (10% chance)
if random.randrange(0, 100) < 10:
# do this just once in a while (10% chance)
self.clear_buckets(buckets)
pipe.execute()