diff --git a/.travis.yml b/.travis.yml index 094f0de0..ccb9a6a0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,7 @@ dist: "bionic" services: - mysql + - redis-server python: - '2.7' @@ -23,6 +24,7 @@ install: before_script: - pip install coverage - pip install codecov + - pip install redis before_install: - mysql -e 'create database pydal;' diff --git a/appveyor.yml b/appveyor.yml index c90f027e..a8918b2f 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,4 +1,8 @@ build: false +before_build: + - choco install redis-64 + - redis-server --service-install + - redis-server --service-start environment: matrix: @@ -26,7 +30,7 @@ init: install: - python -m ensurepip - - pip install codecov + - pip install codecov redis - git submodule update --init --recursive # Check that we have the expected version and architecture for Python - "python --version" diff --git a/gluon/contrib/redis_session.py b/gluon/contrib/redis_session.py index 4ea11442..6413b52a 100644 --- a/gluon/contrib/redis_session.py +++ b/gluon/contrib/redis_session.py @@ -14,6 +14,7 @@ from gluon.storage import Storage from gluon.contrib.redis_utils import acquire_lock, release_lock from gluon.contrib.redis_utils import register_release_lock from gluon._compat import to_native +from datetime import datetime logger = logging.getLogger("web2py.session.redis") @@ -65,13 +66,13 @@ class RedisClient(object): def Field(self, fieldname, type='string', length=None, default=None, required=False, requires=None): - return None + return fieldname, type def define_table(self, tablename, *fields, **args): if not self.tablename: self.tablename = MockTable( self, self.r_server, tablename, self.session_expiry, - self.with_lock) + with_lock=self.with_lock, fields=fields) return self.tablename def __getitem__(self, key): @@ -85,10 +86,26 @@ class RedisClient(object): # this is only called by session2trash.py pass + def convert_dict_string(self, dict_string): + fields = self.tablename.fields + typed_dict = dict() + converters = { + 'boolean': lambda x: 1 if x.decode() == '1' else 0, + 'blob': lambda x: x, + } + for field, ftype in fields: + if field not in dict_string: + continue + if ftype in converters: + typed_dict[field] = converters[ftype](dict_string[field]) + else: + typed_dict[field] = dict_string[field].decode() + return typed_dict + class MockTable(object): - def __init__(self, db, r_server, tablename, session_expiry, with_lock=False): + def __init__(self, db, r_server, tablename, session_expiry, with_lock=False, fields=None): # here self.db is the RedisClient instance self.db = db self.tablename = tablename @@ -101,6 +118,7 @@ class MockTable(object): # remember the session_expiry setting self.session_expiry = session_expiry self.with_lock = with_lock + self.fields = fields if fields is not None else [] def __call__(self, record_id, unique_key=None): # Support DAL shortcut query: table(record_id) @@ -172,7 +190,11 @@ class MockQuery(object): self.value = value self.op = op - def __gt__(self, value, op='ge'): + def __ge__(self, value, op='ge'): + self.value = value + self.op = op + + def __gt__(self, value, op='gt'): self.value = value self.op = op @@ -182,7 +204,7 @@ class MockQuery(object): key = self.keyprefix + ':' + str(self.value) if self.with_lock: acquire_lock(self.db.r_server, key + ':lock', self.value, 2) - rtn = {to_native(k.decode): v for k, v in self.db.r_server.hgetall(key).items()} + rtn = {to_native(k): v for k, v in self.db.r_server.hgetall(key).items()} if rtn: if self.unique_key: # make sure the id and unique_key are correct @@ -190,8 +212,8 @@ class MockQuery(object): rtn['update_record'] = self.update # update record support else: rtn = None - return [Storage(rtn)] if rtn else [] - elif self.op == 'ge' and self.field == 'id' and self.value == 0: + return [Storage(self.db.convert_dict_string(rtn))] if rtn else [] + elif self.op in ('ge', 'gt') and self.field == 'id' and self.value == 0: # means that someone wants the complete list rtn = [] id_idx = "%s:id_idx" % self.keyprefix @@ -204,7 +226,7 @@ class MockQuery(object): # clean up the idx, because the key expired self.db.r_server.srem(id_idx, sess) continue - val = Storage(val) + val = Storage(self.db.convert_dict_string(val)) # add a delete_record method (necessary for sessions2trash.py) val.delete_record = RecordDeleter( self.db, sess, self.keyprefix) diff --git a/gluon/tests/__init__.py b/gluon/tests/__init__.py index 6b3c5480..97aa6595 100644 --- a/gluon/tests/__init__.py +++ b/gluon/tests/__init__.py @@ -8,6 +8,7 @@ from .test_dal import * from .test_cache import * from .test_html import * from .test_contribs import * +from .test_redis import * from .test_routes import * from .test_router import * from .test_authapi import * diff --git a/gluon/tests/test_redis.py b/gluon/tests/test_redis.py new file mode 100644 index 00000000..201cab1d --- /dev/null +++ b/gluon/tests/test_redis.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +""" Unit tests for redis """ + +import unittest +from datetime import datetime + +from gluon._compat import to_bytes, pickle +from gluon.storage import Storage +from gluon.utils import web2py_uuid +from gluon.globals import Request, Response, Session, current +from gluon.contrib.redis_utils import RConn +from gluon.contrib.redis_session import RedisSession +from gluon.contrib.redis_cache import RedisCache + + +class TestRedis(unittest.TestCase): + """ Tests the Redis contrib packages """ + def setUp(self): + request = Request(env={}) + request.application = 'a' + request.controller = 'c' + request.function = 'f' + request.folder = 'applications/admin' + response = Response() + session = Session() + session.connect(request, response) + from gluon.globals import current + current.request = request + current.response = response + current.session = session + self.current = current + rconn = RConn(host='localhost') + self.db = RedisSession(redis_conn=rconn, session_expiry=False) + self.tname = 'testtablename' + return current + + def test_0_redis_session(self): + """ Basic redis read-write """ + db = self.db + response = self.current.response + Field = db.Field + db.define_table( + self.tname, + Field('locked', 'boolean', default=False), + Field('client_ip', length=64), + Field('created_datetime', 'datetime', + default=datetime.now().isoformat()), + Field('modified_datetime', 'datetime'), + Field('unique_key', length=64), + Field('session_data', 'blob'), + ) + table = db[self.tname] + unique_key = web2py_uuid() + dd = dict( + locked=0, + client_ip=response.session_client, + modified_datetime=datetime.now().isoformat(), + unique_key=unique_key, + session_data=pickle.dumps({'test': 123, 'me': 112312312}, pickle.HIGHEST_PROTOCOL) + ) + record_id = table.insert(**dd) + data_from_db = db(table.id == record_id).select()[0] + self.assertDictEqual(Storage(dd), data_from_db, 'get inserted dict') + + dd['locked'] = 1 + table._db(table.id == record_id).update(**dd) + data_from_db = db(table.id == record_id).select()[0] + self.assertDictEqual(Storage(dd), data_from_db, 'get the updated value') + + def test_1_redis_delete(self): + """ Redis session get and delete sessions """ + db = self.db + table = db[self.tname] + all_sessions = db(table.id > 0).select() + self.assertIsNotNone(all_sessions, 'we must have some keys in db') + + for entry in all_sessions: + res = entry.delete_record() + self.assertIsNone(res, 'delete should return None') + + empty_sessions = db(table.id > 0).select() + self.assertEqual(empty_sessions, [], 'no sessions left') +