new redis toolset to use with web2py
This is a refactor of everything web2py uses with redis. Specifically: - a refactored redis_cache.py that fixes #958, allowing a "fail gracefully" behaviour in case redis is not available - a refactored redis_session.py that blocks less with with_lock=True (although still not optimal) - a new (and NEEDED) redis_utils.py that serves as the base for everything else, allowing an RConn object that you can freely use as a redis.StrictRedis connection, and that you can override in case you're using a different library (or that web2py will use in case redis-py won't be the de-facto standard around) - a newly - and much anticipated - redis_scheduler.py. It's a slip-in replacement for the standard scheduler that uses redis for workers coordination. Feel free to dig in the code and improve it. For redis_cache and redis_session changes are BREAKING. It means that users will need to change the import locations and tune a bit the code. Now every module depends on an gluon.contrib.redis_utils.RConn object (or similar) that in turns is very similar to a redis.StrictRedis one. The redis instance is EXTERNAL to the modules themselves (no more "host, port, db, password" parameters in RedisCache or RedisSession) See the relevant docstrings for usage examples
This commit is contained in:
niphlod
@@ -1,13 +1,18 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Developed by niphlod@gmail.com
|
||||
License MIT/BSD/GPL
|
||||
|
||||
Redis-backed sessions
|
||||
"""
|
||||
|
||||
import redis
|
||||
from gluon import current
|
||||
from gluon.storage import Storage
|
||||
import time
|
||||
import logging
|
||||
import thread
|
||||
from gluon import current
|
||||
from gluon.storage import Storage
|
||||
from gluon.contrib.redis_utils import acquire_lock, release_lock
|
||||
from gluon.contrib.redis_utils import register_release_lock
|
||||
|
||||
logger = logging.getLogger("web2py.session.redis")
|
||||
|
||||
@@ -16,10 +21,20 @@ locker = thread.allocate_lock()
|
||||
|
||||
def RedisSession(*args, **vars):
|
||||
"""
|
||||
Usage example: put in models
|
||||
from gluon.contrib.redis_session import RedisSession
|
||||
sessiondb = RedisSession('localhost:6379',db=0, session_expiry=False, password=None)
|
||||
session.connect(request, response, db = sessiondb)
|
||||
Usage example: put in models::
|
||||
|
||||
from gluon.contrib.redis_utils import RConn
|
||||
rconn = RConn()
|
||||
from gluon.contrib.redis_session
|
||||
sessiondb = RedisSession(redis_conn=rconn, with_lock=True, session_expiry=False)
|
||||
session.connect(request, response, db = sessiondb)
|
||||
|
||||
Args:
|
||||
redis_conn: a redis-like connection object
|
||||
with_lock: prevent concurrent modifications to the same session
|
||||
session_expiry: delete automatically sessions after n seconds
|
||||
(still need to run sessions2trash.py every 1M sessions
|
||||
or so)
|
||||
|
||||
Simple slip-in storage for session
|
||||
"""
|
||||
@@ -36,30 +51,9 @@ def RedisSession(*args, **vars):
|
||||
|
||||
class RedisClient(object):
|
||||
|
||||
meta_storage = {}
|
||||
MAX_RETRIES = 5
|
||||
RETRIES = 0
|
||||
_release_script = None
|
||||
|
||||
def __init__(self, server='localhost:6379', db=None, debug=False,
|
||||
session_expiry=False, with_lock=False, password=None):
|
||||
"""session_expiry can be an integer, in seconds, to set the default expiration
|
||||
of sessions. The corresponding record will be deleted from the redis instance,
|
||||
and there's virtually no need to run sessions2trash.py
|
||||
"""
|
||||
self.server = server
|
||||
self.password = password
|
||||
self.db = db or 0
|
||||
host, port = (self.server.split(':') + ['6379'])[:2]
|
||||
port = int(port)
|
||||
self.debug = debug
|
||||
if current and current.request:
|
||||
self.app = current.request.application
|
||||
else:
|
||||
self.app = ''
|
||||
self.r_server = redis.Redis(host=host, port=port, db=self.db, password=self.password)
|
||||
if with_lock:
|
||||
RedisClient._release_script = self.r_server.register_script(_LUA_RELEASE_LOCK)
|
||||
def __init__(self, redis_conn, session_expiry=False, with_lock=False):
|
||||
self.r_server = redis_conn
|
||||
self._release_script = register_release_lock(self.r_server)
|
||||
self.tablename = None
|
||||
self.session_expiry = session_expiry
|
||||
self.with_lock = with_lock
|
||||
@@ -93,12 +87,11 @@ class RedisClient(object):
|
||||
class MockTable(object):
|
||||
|
||||
def __init__(self, db, r_server, tablename, session_expiry, with_lock=False):
|
||||
# here self.db is the RedisClient instance
|
||||
self.db = db
|
||||
self.r_server = r_server
|
||||
self.tablename = tablename
|
||||
# set the namespace for sessions of this app
|
||||
self.keyprefix = 'w2p:sess:%s' % tablename.replace(
|
||||
'web2py_session_', '')
|
||||
self.keyprefix = 'w2p:sess:%s' % tablename.replace('web2py_session_', '')
|
||||
# fast auto-increment id (needed for session handling)
|
||||
self.serial = "%s:serial" % self.keyprefix
|
||||
# index of all the session keys of this app
|
||||
@@ -126,7 +119,7 @@ class MockTable(object):
|
||||
if key == 'id':
|
||||
# return a fake query. We need to query it just by id for normal operations
|
||||
self.query = MockQuery(
|
||||
field='id', db=self.r_server,
|
||||
field='id', db=self.db,
|
||||
prefix=self.keyprefix, session_expiry=self.session_expiry,
|
||||
with_lock=self.with_lock, unique_key=self.unique_key
|
||||
)
|
||||
@@ -140,12 +133,12 @@ class MockTable(object):
|
||||
# 'locked', 'client_ip','created_datetime','modified_datetime'
|
||||
# 'unique_key', 'session_data'
|
||||
# retrieve a new key
|
||||
newid = str(self.r_server.incr(self.serial))
|
||||
newid = str(self.db.r_server.incr(self.serial))
|
||||
key = self.keyprefix + ':' + newid
|
||||
if self.with_lock:
|
||||
key_lock = key + ':lock'
|
||||
acquire_lock(self.r_server, key_lock, newid)
|
||||
with self.r_server.pipeline() as pipe:
|
||||
acquire_lock(self.db.r_server, key_lock, newid)
|
||||
with self.db.r_server.pipeline() as pipe:
|
||||
# add it to the index
|
||||
pipe.sadd(self.id_idx, key)
|
||||
# set a hash key with the Storage
|
||||
@@ -154,7 +147,7 @@ class MockTable(object):
|
||||
pipe.expire(key, self.session_expiry)
|
||||
pipe.execute()
|
||||
if self.with_lock:
|
||||
release_lock(self.r_server, key_lock, newid)
|
||||
release_lock(self.db, key_lock, newid)
|
||||
return newid
|
||||
|
||||
|
||||
@@ -186,8 +179,8 @@ class MockQuery(object):
|
||||
# means that someone wants to retrieve the key self.value
|
||||
key = self.keyprefix + ':' + str(self.value)
|
||||
if self.with_lock:
|
||||
acquire_lock(self.db, key + ':lock', self.value)
|
||||
rtn = self.db.hgetall(key)
|
||||
acquire_lock(self.db.r_server, key + ':lock', self.value, 2)
|
||||
rtn = self.db.r_server.hgetall(key)
|
||||
if rtn:
|
||||
if self.unique_key:
|
||||
# make sure the id and unique_key are correct
|
||||
@@ -201,13 +194,13 @@ class MockQuery(object):
|
||||
rtn = []
|
||||
id_idx = "%s:id_idx" % self.keyprefix
|
||||
# find all session keys of this app
|
||||
allkeys = self.db.smembers(id_idx)
|
||||
allkeys = self.db.r_server.smembers(id_idx)
|
||||
for sess in allkeys:
|
||||
val = self.db.hgetall(sess)
|
||||
val = self.db.r_server.hgetall(sess)
|
||||
if not val:
|
||||
if self.session_expiry:
|
||||
# clean up the idx, because the key expired
|
||||
self.db.srem(id_idx, sess)
|
||||
self.db.r_server.srem(id_idx, sess)
|
||||
continue
|
||||
val = Storage(val)
|
||||
# add a delete_record method (necessary for sessions2trash.py)
|
||||
@@ -222,9 +215,9 @@ class MockQuery(object):
|
||||
# means that the session has been found and needs an update
|
||||
if self.op == 'eq' and self.field == 'id' and self.value:
|
||||
key = self.keyprefix + ':' + str(self.value)
|
||||
if not self.db.exists(key):
|
||||
if not self.db.r_server.exists(key):
|
||||
return None
|
||||
with self.db.pipeline() as pipe:
|
||||
with self.db.r_server.pipeline() as pipe:
|
||||
pipe.hmset(key, kwargs)
|
||||
if self.session_expiry:
|
||||
pipe.expire(key, self.session_expiry)
|
||||
@@ -238,7 +231,7 @@ class MockQuery(object):
|
||||
if self.op == 'eq' and self.field == 'id' and self.value:
|
||||
id_idx = "%s:id_idx" % self.keyprefix
|
||||
key = self.keyprefix + ':' + str(self.value)
|
||||
with self.db.pipeline() as pipe:
|
||||
with self.db.r_server.pipeline() as pipe:
|
||||
pipe.delete(key)
|
||||
pipe.srem(id_idx, key)
|
||||
rtn = pipe.execute()
|
||||
@@ -254,29 +247,6 @@ class RecordDeleter(object):
|
||||
def __call__(self):
|
||||
id_idx = "%s:id_idx" % self.keyprefix
|
||||
# remove from the index
|
||||
self.db.srem(id_idx, self.key)
|
||||
self.db.r_server.srem(id_idx, self.key)
|
||||
# remove the key itself
|
||||
self.db.delete(self.key)
|
||||
|
||||
|
||||
def acquire_lock(conn, lockname, identifier, ltime=10):
|
||||
while True:
|
||||
if conn.set(lockname, identifier, ex=ltime, nx=True):
|
||||
return identifier
|
||||
time.sleep(.01)
|
||||
|
||||
|
||||
_LUA_RELEASE_LOCK = """
|
||||
if redis.call("get", KEYS[1]) == ARGV[1]
|
||||
then
|
||||
return redis.call("del", KEYS[1])
|
||||
else
|
||||
return 0
|
||||
end
|
||||
"""
|
||||
|
||||
|
||||
def release_lock(conn, lockname, identifier):
|
||||
return RedisClient._release_script(
|
||||
keys=[lockname], args=[identifier],
|
||||
client=conn)
|
||||
self.db.r_server.delete(self.key)
|
||||
|
||||
Reference in New Issue
Block a user