From 517f88891f97d870af8fd94886b7dca63689d261 Mon Sep 17 00:00:00 2001
From: Dinis <dinis@lage.pw>
Date: Thu, 5 Dec 2019 19:26:40 +0000
Subject: [PATCH 01/10] Add a test that simulates redis session usage.

---
 gluon/tests/test_redis.py | 78 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 78 insertions(+)
 create mode 100644 gluon/tests/test_redis.py

diff --git a/gluon/tests/test_redis.py b/gluon/tests/test_redis.py
new file mode 100644
index 00000000..41f76441
--- /dev/null
+++ b/gluon/tests/test_redis.py
@@ -0,0 +1,78 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+""" Unit tests for redis """
+
+import unittest
+import os
+import time
+from datetime import datetime
+
+from gluon._compat import to_bytes
+from gluon.storage import Storage
+from gluon.utils import web2py_uuid
+from gluon.globals import Request, Response, Session, current
+from gluon.contrib.redis_utils import RConn
+from gluon.contrib.redis_session import RedisSession
+from gluon.contrib.redis_cache import RedisCache
+
+
+def setup_clean_session():
+    request = Request(env={})
+    request.application = 'a'
+    request.controller = 'c'
+    request.function = 'f'
+    request.folder = 'applications/admin'
+    response = Response()
+    session = Session()
+    session.connect(request, response)
+    from gluon.globals import current
+    current.request = request
+    current.response = response
+    current.session = session
+    return current
+
+
+def setUpModule():
+    pass
+
+
+def tearDownModule():
+    if os.path.isfile('appconfig.json'):
+        os.unlink('appconfig.json')
+
+
+class TestRedis(unittest.TestCase):
+    """ Tests the Redis contrib packages """
+
+    def test_0_redis_session(self):
+        """ Basic redis read-write """
+        current = setup_clean_session()
+        response = current.response
+        rconn = RConn(host='redis')
+        db = RedisSession(redis_conn=rconn, session_expiry=False)
+        tname = 'testtablename'
+        Field = db.Field
+        db.define_table(
+            tname,
+            Field('locked', 'boolean', default=False),
+            Field('client_ip', length=64),
+            Field('created_datetime', 'datetime',
+                  default=datetime.now),
+            Field('modified_datetime', 'datetime'),
+            Field('unique_key', length=64),
+            Field('session_data', 'blob'),
+        )
+        table = db[tname]
+        unique_key = web2py_uuid()
+        dd = dict(
+            locked=0,
+            client_ip=response.session_client,
+            modified_datetime=datetime.now,
+            unique_key=unique_key
+        )
+        record_id = table.insert(**dd)
+        data_from_db = db(table.id == record_id).select()
+        #print('data_from_db=', data_from_db)
+        self.assertDictEqual(dd, data_from_db)
+

From bad0d0b26bd66244245d7a661e413ebee062056e Mon Sep 17 00:00:00 2001
From: Dinis <dinis@lage.pw>
Date: Thu, 5 Dec 2019 19:53:20 +0000
Subject: [PATCH 02/10] Start new testing of redis_session and save redis
 session table information so we can recover data from redis in the native
 format.

---
 gluon/contrib/redis_session.py |  9 +++++----
 gluon/tests/test_redis.py      | 20 +++++---------------
 2 files changed, 10 insertions(+), 19 deletions(-)

diff --git a/gluon/contrib/redis_session.py b/gluon/contrib/redis_session.py
index 4ea11442..3c73bcf3 100644
--- a/gluon/contrib/redis_session.py
+++ b/gluon/contrib/redis_session.py
@@ -65,13 +65,13 @@ class RedisClient(object):
 
     def Field(self, fieldname, type='string', length=None, default=None,
               required=False, requires=None):
-        return None
+        return fieldname, type
 
     def define_table(self, tablename, *fields, **args):
         if not self.tablename:
             self.tablename = MockTable(
                 self, self.r_server, tablename, self.session_expiry,
-                self.with_lock)
+                self.with_lock, fields=fields)
         return self.tablename
 
     def __getitem__(self, key):
@@ -88,7 +88,7 @@ class RedisClient(object):
 
 class MockTable(object):
 
-    def __init__(self, db, r_server, tablename, session_expiry, with_lock=False):
+    def __init__(self, db, r_server, tablename, session_expiry, with_lock=False, fields=None):
         # here self.db is the RedisClient instance
         self.db = db
         self.tablename = tablename
@@ -101,6 +101,7 @@ class MockTable(object):
         # remember the session_expiry setting
         self.session_expiry = session_expiry
         self.with_lock = with_lock
+        self.fields = fields if fields is not None else []
 
     def __call__(self, record_id, unique_key=None):
         # Support DAL shortcut query: table(record_id)
@@ -182,7 +183,7 @@ class MockQuery(object):
             key = self.keyprefix + ':' + str(self.value)
             if self.with_lock:
                 acquire_lock(self.db.r_server, key + ':lock', self.value, 2)
-            rtn = {to_native(k.decode): v for k, v in self.db.r_server.hgetall(key).items()}
+            rtn = {to_native(k): v for k, v in self.db.r_server.hgetall(key).items()}
             if rtn:
                 if self.unique_key:
                     # make sure the id and unique_key are correct
diff --git a/gluon/tests/test_redis.py b/gluon/tests/test_redis.py
index 41f76441..7b6b65f8 100644
--- a/gluon/tests/test_redis.py
+++ b/gluon/tests/test_redis.py
@@ -52,27 +52,17 @@ class TestRedis(unittest.TestCase):
         rconn = RConn(host='redis')
         db = RedisSession(redis_conn=rconn, session_expiry=False)
         tname = 'testtablename'
-        Field = db.Field
-        db.define_table(
-            tname,
-            Field('locked', 'boolean', default=False),
-            Field('client_ip', length=64),
-            Field('created_datetime', 'datetime',
-                  default=datetime.now),
-            Field('modified_datetime', 'datetime'),
-            Field('unique_key', length=64),
-            Field('session_data', 'blob'),
-        )
+        db.define_table(tname)
         table = db[tname]
         unique_key = web2py_uuid()
         dd = dict(
             locked=0,
             client_ip=response.session_client,
-            modified_datetime=datetime.now,
+            modified_datetime=datetime.now().isoformat(),
             unique_key=unique_key
         )
         record_id = table.insert(**dd)
-        data_from_db = db(table.id == record_id).select()
-        #print('data_from_db=', data_from_db)
-        self.assertDictEqual(dd, data_from_db)
+        data_from_db = db(table.id == record_id).select()[0]
+        print('data_from_db=', data_from_db)
+        self.assertDictEqual(Storage(dd), data_from_db)
 

From ce917feb7e2d81c6605ce24117b818bd3529d4e0 Mon Sep 17 00:00:00 2001
From: Dinis <dinis@lage.pw>
Date: Wed, 11 Dec 2019 18:31:44 +0000
Subject: [PATCH 03/10] Fix redis_session types for new redis client. Add
 testing in travis and appveyor.yml

---
 .travis.yml                    |  1 +
 appveyor.yml                   |  6 +++++-
 gluon/contrib/redis_session.py | 17 +++++++++++++++--
 gluon/tests/test_redis.py      | 24 +++++++++++++++++++-----
 4 files changed, 40 insertions(+), 8 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 094f0de0..65f28d4e 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -6,6 +6,7 @@ dist: "bionic"
 
 services:
   - mysql
+  - redis-server
 
 python:
   - '2.7'
diff --git a/appveyor.yml b/appveyor.yml
index c90f027e..a8918b2f 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -1,4 +1,8 @@
 build: false
+before_build:
+  - choco install redis-64
+  - redis-server --service-install
+  - redis-server --service-start
 
 environment:
   matrix:
@@ -26,7 +30,7 @@ init:
 
 install:
   - python -m ensurepip
-  - pip install codecov
+  - pip install codecov redis
   - git submodule update --init --recursive
   # Check that we have the expected version and architecture for Python
   - "python --version"
diff --git a/gluon/contrib/redis_session.py b/gluon/contrib/redis_session.py
index 3c73bcf3..025af283 100644
--- a/gluon/contrib/redis_session.py
+++ b/gluon/contrib/redis_session.py
@@ -14,6 +14,7 @@ from gluon.storage import Storage
 from gluon.contrib.redis_utils import acquire_lock, release_lock
 from gluon.contrib.redis_utils import register_release_lock
 from gluon._compat import to_native
+from datetime import datetime
 
 logger = logging.getLogger("web2py.session.redis")
 
@@ -71,7 +72,7 @@ class RedisClient(object):
         if not self.tablename:
             self.tablename = MockTable(
                 self, self.r_server, tablename, self.session_expiry,
-                self.with_lock, fields=fields)
+                with_lock=self.with_lock, fields=fields)
         return self.tablename
 
     def __getitem__(self, key):
@@ -85,6 +86,18 @@ class RedisClient(object):
         # this is only called by session2trash.py
         pass
 
+    def convert_dict_string(self, dict_string):
+        fields = self.tablename.fields
+        typed_dict = dict()
+        converters = {
+            'boolean': lambda(x): 1 if x == '1' else 0,
+        }
+        for field, ftype in fields:
+            if field not in dict_string:
+                continue
+            typed_dict[field] = converters[ftype](dict_string[field]) if ftype in converters else dict_string[field]
+        return typed_dict
+
 
 class MockTable(object):
 
@@ -191,7 +204,7 @@ class MockQuery(object):
                         rtn['update_record'] = self.update  # update record support
                     else:
                         rtn = None
-            return [Storage(rtn)] if rtn else []
+            return [Storage(self.db.convert_dict_string(rtn))] if rtn else []
         elif self.op == 'ge' and self.field == 'id' and self.value == 0:
             # means that someone wants the complete list
             rtn = []
diff --git a/gluon/tests/test_redis.py b/gluon/tests/test_redis.py
index 7b6b65f8..ed4bbaf9 100644
--- a/gluon/tests/test_redis.py
+++ b/gluon/tests/test_redis.py
@@ -8,7 +8,7 @@ import os
 import time
 from datetime import datetime
 
-from gluon._compat import to_bytes
+from gluon._compat import to_bytes, pickle
 from gluon.storage import Storage
 from gluon.utils import web2py_uuid
 from gluon.globals import Request, Response, Session, current
@@ -49,20 +49,34 @@ class TestRedis(unittest.TestCase):
         """ Basic redis read-write """
         current = setup_clean_session()
         response = current.response
-        rconn = RConn(host='redis')
+        rconn = RConn(host='localhost')
         db = RedisSession(redis_conn=rconn, session_expiry=False)
         tname = 'testtablename'
-        db.define_table(tname)
+        Field = db.Field
+        db.define_table(
+            tname,
+            Field('locked', 'boolean', default=False),
+            Field('client_ip', length=64),
+            Field('created_datetime', 'datetime',
+                  default=datetime.now().isoformat()),
+            Field('modified_datetime', 'datetime'),
+            Field('unique_key', length=64),
+            Field('session_data', 'blob'),
+        )
         table = db[tname]
         unique_key = web2py_uuid()
         dd = dict(
             locked=0,
             client_ip=response.session_client,
             modified_datetime=datetime.now().isoformat(),
-            unique_key=unique_key
+            unique_key=unique_key,
+            session_data=pickle.dumps({'test': 123, 'me': 112312312}, pickle.HIGHEST_PROTOCOL)
         )
         record_id = table.insert(**dd)
         data_from_db = db(table.id == record_id).select()[0]
-        print('data_from_db=', data_from_db)
         self.assertDictEqual(Storage(dd), data_from_db)
 
+        dd['locked'] = 1
+        table._db(table.id == record_id).update(**dd)
+        data_from_db = db(table.id == record_id).select()[0]
+        self.assertDictEqual(Storage(dd), data_from_db)

From 981254ec61d32a0f739152fa594b5dc505a74061 Mon Sep 17 00:00:00 2001
From: Dinis <dinis@lage.pw>
Date: Wed, 11 Dec 2019 18:55:14 +0000
Subject: [PATCH 04/10] add test_redis to gluon init.

---
 gluon/tests/__init__.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/gluon/tests/__init__.py b/gluon/tests/__init__.py
index 6b3c5480..97aa6595 100644
--- a/gluon/tests/__init__.py
+++ b/gluon/tests/__init__.py
@@ -8,6 +8,7 @@ from .test_dal import *
 from .test_cache import *
 from .test_html import *
 from .test_contribs import *
+from .test_redis import *
 from .test_routes import *
 from .test_router import *
 from .test_authapi import *

From 223755d89460db6a91af90add3ce2dd280267aaa Mon Sep 17 00:00:00 2001
From: Dinis <dinis@lage.pw>
Date: Wed, 11 Dec 2019 19:06:19 +0000
Subject: [PATCH 05/10] fix missing pip install redis.

---
 .travis.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.travis.yml b/.travis.yml
index 65f28d4e..ccb9a6a0 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -24,6 +24,7 @@ install:
 before_script:
   - pip install coverage
   - pip install codecov
+  - pip install redis
 
 before_install:
   - mysql -e 'create database pydal;'

From 8f84b5df34c7033adf24f0ac79cb1017fcfe8a76 Mon Sep 17 00:00:00 2001
From: Dinis <dinis@lage.pw>
Date: Wed, 11 Dec 2019 19:09:51 +0000
Subject: [PATCH 06/10] fix lambda definition.

---
 gluon/contrib/redis_session.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/gluon/contrib/redis_session.py b/gluon/contrib/redis_session.py
index 025af283..22270ad3 100644
--- a/gluon/contrib/redis_session.py
+++ b/gluon/contrib/redis_session.py
@@ -90,7 +90,7 @@ class RedisClient(object):
         fields = self.tablename.fields
         typed_dict = dict()
         converters = {
-            'boolean': lambda(x): 1 if x == '1' else 0,
+            'boolean': lambda x: 1 if x == '1' else 0,
         }
         for field, ftype in fields:
             if field not in dict_string:

From 022ddd49c4535aa200d52ff0bcb6ea2bfd035758 Mon Sep 17 00:00:00 2001
From: Dinis <dinis@lage.pw>
Date: Thu, 12 Dec 2019 14:37:41 +0000
Subject: [PATCH 07/10] fixed convertions for compatibility with Python3. Add
 further testing of redis sessions functions.

---
 gluon/contrib/redis_session.py | 18 +++++++++++++-----
 gluon/tests/test_redis.py      | 14 ++++++++++++--
 2 files changed, 25 insertions(+), 7 deletions(-)

diff --git a/gluon/contrib/redis_session.py b/gluon/contrib/redis_session.py
index 22270ad3..6413b52a 100644
--- a/gluon/contrib/redis_session.py
+++ b/gluon/contrib/redis_session.py
@@ -90,12 +90,16 @@ class RedisClient(object):
         fields = self.tablename.fields
         typed_dict = dict()
         converters = {
-            'boolean': lambda x: 1 if x == '1' else 0,
+            'boolean': lambda x: 1 if x.decode() == '1' else 0,
+            'blob': lambda x: x,
         }
         for field, ftype in fields:
             if field not in dict_string:
                 continue
-            typed_dict[field] = converters[ftype](dict_string[field]) if ftype in converters else dict_string[field]
+            if ftype in converters:
+                typed_dict[field] = converters[ftype](dict_string[field])
+            else:
+                typed_dict[field] = dict_string[field].decode()
         return typed_dict
 
 
@@ -186,7 +190,11 @@ class MockQuery(object):
         self.value = value
         self.op = op
 
-    def __gt__(self, value, op='ge'):
+    def __ge__(self, value, op='ge'):
+        self.value = value
+        self.op = op
+
+    def __gt__(self, value, op='gt'):
         self.value = value
         self.op = op
 
@@ -205,7 +213,7 @@ class MockQuery(object):
                     else:
                         rtn = None
             return [Storage(self.db.convert_dict_string(rtn))] if rtn else []
-        elif self.op == 'ge' and self.field == 'id' and self.value == 0:
+        elif self.op in ('ge', 'gt') and self.field == 'id' and self.value == 0:
             # means that someone wants the complete list
             rtn = []
             id_idx = "%s:id_idx" % self.keyprefix
@@ -218,7 +226,7 @@ class MockQuery(object):
                         # clean up the idx, because the key expired
                         self.db.r_server.srem(id_idx, sess)
                     continue
-                val = Storage(val)
+                val = Storage(self.db.convert_dict_string(val))
                 # add a delete_record method (necessary for sessions2trash.py)
                 val.delete_record = RecordDeleter(
                     self.db, sess, self.keyprefix)
diff --git a/gluon/tests/test_redis.py b/gluon/tests/test_redis.py
index ed4bbaf9..ce7dc3b7 100644
--- a/gluon/tests/test_redis.py
+++ b/gluon/tests/test_redis.py
@@ -74,9 +74,19 @@ class TestRedis(unittest.TestCase):
         )
         record_id = table.insert(**dd)
         data_from_db = db(table.id == record_id).select()[0]
-        self.assertDictEqual(Storage(dd), data_from_db)
+        self.assertDictEqual(Storage(dd), data_from_db, 'get inserted dict')
 
         dd['locked'] = 1
         table._db(table.id == record_id).update(**dd)
         data_from_db = db(table.id == record_id).select()[0]
-        self.assertDictEqual(Storage(dd), data_from_db)
+        self.assertDictEqual(Storage(dd), data_from_db, 'get the updated value')
+
+        all_sessions = db(table.id > 0).select()
+        self.assertIsNotNone(all_sessions, 'we must have some keys in db')
+
+        for entry in all_sessions:
+            res = entry.delete_record()
+            self.assertIsNone(res, 'delete should return None')
+
+        empty_sessions = db(table.id > 0).select()
+        self.assertEqual(empty_sessions, [], 'no sessions left')

From 3272755fea2d0a9daeec2f9688d350316cdb4d0e Mon Sep 17 00:00:00 2001
From: Dinis <dinis@lage.pw>
Date: Thu, 12 Dec 2019 15:00:37 +0000
Subject: [PATCH 08/10] better structure in the redis test class.

---
 gluon/tests/test_redis.py | 59 ++++++++++++++++++---------------------
 1 file changed, 27 insertions(+), 32 deletions(-)

diff --git a/gluon/tests/test_redis.py b/gluon/tests/test_redis.py
index ce7dc3b7..ff6dfd11 100644
--- a/gluon/tests/test_redis.py
+++ b/gluon/tests/test_redis.py
@@ -17,44 +17,34 @@ from gluon.contrib.redis_session import RedisSession
 from gluon.contrib.redis_cache import RedisCache
 
 
-def setup_clean_session():
-    request = Request(env={})
-    request.application = 'a'
-    request.controller = 'c'
-    request.function = 'f'
-    request.folder = 'applications/admin'
-    response = Response()
-    session = Session()
-    session.connect(request, response)
-    from gluon.globals import current
-    current.request = request
-    current.response = response
-    current.session = session
-    return current
-
-
-def setUpModule():
-    pass
-
-
-def tearDownModule():
-    if os.path.isfile('appconfig.json'):
-        os.unlink('appconfig.json')
-
-
 class TestRedis(unittest.TestCase):
     """ Tests the Redis contrib packages """
+    def setUp(self):
+        request = Request(env={})
+        request.application = 'a'
+        request.controller = 'c'
+        request.function = 'f'
+        request.folder = 'applications/admin'
+        response = Response()
+        session = Session()
+        session.connect(request, response)
+        from gluon.globals import current
+        current.request = request
+        current.response = response
+        current.session = session
+        self.current = current
+        rconn = RConn(host='redis')
+        self.db = RedisSession(redis_conn=rconn, session_expiry=False)
+        self.tname = 'testtablename'
+        return current
 
     def test_0_redis_session(self):
         """ Basic redis read-write """
-        current = setup_clean_session()
-        response = current.response
-        rconn = RConn(host='localhost')
-        db = RedisSession(redis_conn=rconn, session_expiry=False)
-        tname = 'testtablename'
+        db = self.db
+        response = self.current.response
         Field = db.Field
         db.define_table(
-            tname,
+            self.tname,
             Field('locked', 'boolean', default=False),
             Field('client_ip', length=64),
             Field('created_datetime', 'datetime',
@@ -63,7 +53,7 @@ class TestRedis(unittest.TestCase):
             Field('unique_key', length=64),
             Field('session_data', 'blob'),
         )
-        table = db[tname]
+        table = db[self.tname]
         unique_key = web2py_uuid()
         dd = dict(
             locked=0,
@@ -81,6 +71,10 @@ class TestRedis(unittest.TestCase):
         data_from_db = db(table.id == record_id).select()[0]
         self.assertDictEqual(Storage(dd), data_from_db, 'get the updated value')
 
+    def test_1_redis_delete(self):
+        """ Redis session get and delete sessions """
+        db = self.db
+        table = db[self.tname]
         all_sessions = db(table.id > 0).select()
         self.assertIsNotNone(all_sessions, 'we must have some keys in db')
 
@@ -90,3 +84,4 @@ class TestRedis(unittest.TestCase):
 
         empty_sessions = db(table.id > 0).select()
         self.assertEqual(empty_sessions, [], 'no sessions left')
+

From f5cdf17c481cb66e13e2df3f4bb64acc6d0d8bc0 Mon Sep 17 00:00:00 2001
From: Dinis <dinis@lage.pw>
Date: Thu, 12 Dec 2019 15:07:23 +0000
Subject: [PATCH 09/10] fix wrong redis server host.

---
 gluon/tests/test_redis.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/gluon/tests/test_redis.py b/gluon/tests/test_redis.py
index ff6dfd11..2edae518 100644
--- a/gluon/tests/test_redis.py
+++ b/gluon/tests/test_redis.py
@@ -33,7 +33,7 @@ class TestRedis(unittest.TestCase):
         current.response = response
         current.session = session
         self.current = current
-        rconn = RConn(host='redis')
+        rconn = RConn(host='localhost')
         self.db = RedisSession(redis_conn=rconn, session_expiry=False)
         self.tname = 'testtablename'
         return current

From 2efa54a2d7e9c1e63eaf0118bf4967275d7ccfc3 Mon Sep 17 00:00:00 2001
From: Dinis <dinis@lage.pw>
Date: Thu, 12 Dec 2019 18:03:10 +0000
Subject: [PATCH 10/10] remove useless imports.

---
 gluon/tests/test_redis.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/gluon/tests/test_redis.py b/gluon/tests/test_redis.py
index 2edae518..201cab1d 100644
--- a/gluon/tests/test_redis.py
+++ b/gluon/tests/test_redis.py
@@ -4,8 +4,6 @@
 """ Unit tests for redis """
 
 import unittest
-import os
-import time
 from datetime import datetime
 
 from gluon._compat import to_bytes, pickle