Update dal to the current pydal master with major bugfix of JSON parser

handoff.

.travis.yml

@@ -6,6 +6,7 @@ dist: "bionic"
 
 services:
   - mysql
+  - redis-server
 
 python:
   - '2.7'
@@ -23,6 +24,7 @@ install:
 before_script:
   - pip install coverage
   - pip install codecov
+  - pip install redis
 
 before_install:
   - mysql -e 'create database pydal;'

appveyor.yml

@@ -1,4 +1,8 @@
 build: false
+before_build:
+  - choco install redis-64
+  - redis-server --service-install
+  - redis-server --service-start
 
 environment:
   matrix:
@@ -26,7 +30,7 @@ init:
 
 install:
   - python -m ensurepip
-  - pip install codecov
+  - pip install codecov redis
   - git submodule update --init --recursive
   # Check that we have the expected version and architecture for Python
   - "python --version"

gluon/contrib/redis_session.py

@@ -14,6 +14,7 @@ from gluon.storage import Storage
 from gluon.contrib.redis_utils import acquire_lock, release_lock
 from gluon.contrib.redis_utils import register_release_lock
 from gluon._compat import to_native
+from datetime import datetime
 
 logger = logging.getLogger("web2py.session.redis")
 
@@ -65,13 +66,13 @@ class RedisClient(object):
 
     def Field(self, fieldname, type='string', length=None, default=None,
               required=False, requires=None):
-        return None
+        return fieldname, type
 
     def define_table(self, tablename, *fields, **args):
         if not self.tablename:
             self.tablename = MockTable(
                 self, self.r_server, tablename, self.session_expiry,
-                self.with_lock)
+                with_lock=self.with_lock, fields=fields)
         return self.tablename
 
     def __getitem__(self, key):
@@ -85,10 +86,26 @@ class RedisClient(object):
         # this is only called by session2trash.py
         pass
 
+    def convert_dict_string(self, dict_string):
+        fields = self.tablename.fields
+        typed_dict = dict()
+        converters = {
+            'boolean': lambda x: 1 if x.decode() == '1' else 0,
+            'blob': lambda x: x,
+        }
+        for field, ftype in fields:
+            if field not in dict_string:
+                continue
+            if ftype in converters:
+                typed_dict[field] = converters[ftype](dict_string[field])
+            else:
+                typed_dict[field] = dict_string[field].decode()
+        return typed_dict
+
 
 class MockTable(object):
 
-    def __init__(self, db, r_server, tablename, session_expiry, with_lock=False):
+    def __init__(self, db, r_server, tablename, session_expiry, with_lock=False, fields=None):
         # here self.db is the RedisClient instance
         self.db = db
         self.tablename = tablename
@@ -101,6 +118,7 @@ class MockTable(object):
         # remember the session_expiry setting
         self.session_expiry = session_expiry
         self.with_lock = with_lock
+        self.fields = fields if fields is not None else []
 
     def __call__(self, record_id, unique_key=None):
         # Support DAL shortcut query: table(record_id)
@@ -172,7 +190,11 @@ class MockQuery(object):
         self.value = value
         self.op = op
 
-    def __gt__(self, value, op='ge'):
+    def __ge__(self, value, op='ge'):
+        self.value = value
+        self.op = op
+
+    def __gt__(self, value, op='gt'):
         self.value = value
         self.op = op
 
@@ -182,7 +204,7 @@ class MockQuery(object):
             key = self.keyprefix + ':' + str(self.value)
             if self.with_lock:
                 acquire_lock(self.db.r_server, key + ':lock', self.value, 2)
-            rtn = {to_native(k.decode): v for k, v in self.db.r_server.hgetall(key).items()}
+            rtn = {to_native(k): v for k, v in self.db.r_server.hgetall(key).items()}
             if rtn:
                 if self.unique_key:
                     # make sure the id and unique_key are correct
@@ -190,8 +212,8 @@ class MockQuery(object):
                         rtn['update_record'] = self.update  # update record support
                     else:
                         rtn = None
-            return [Storage(rtn)] if rtn else []
-        elif self.op == 'ge' and self.field == 'id' and self.value == 0:
+            return [Storage(self.db.convert_dict_string(rtn))] if rtn else []
+        elif self.op in ('ge', 'gt') and self.field == 'id' and self.value == 0:
             # means that someone wants the complete list
             rtn = []
             id_idx = "%s:id_idx" % self.keyprefix
@@ -204,7 +226,7 @@ class MockQuery(object):
                         # clean up the idx, because the key expired
                         self.db.r_server.srem(id_idx, sess)
                     continue
-                val = Storage(val)
+                val = Storage(self.db.convert_dict_string(val))
                 # add a delete_record method (necessary for sessions2trash.py)
                 val.delete_record = RecordDeleter(
                     self.db, sess, self.keyprefix)

gluon/sqlhtml.py

@@ -461,7 +461,12 @@ class RadioWidget(OptionsWidget):
             opts.append(child(tds))
 
         if opts:
-            opts[-1][0][0]['hideerror'] = False
+            opts.append(
+                INPUT(requires=attr.get('requires', None),
+                      _style="display:none;",
+                      _disabled="disabled",
+                      _name=field.name,
+                      hideerror=False))
         return parent(*opts, **attr)
 
 
@@ -1814,7 +1819,7 @@ class SQLFORM(FORM):
                 if not field.widget and field.type.startswith('list:') and \
                         not OptionsWidget.has_options(field):
                     field.widget = self.widgets.list.widget
-                if field.widget and fieldname in request_vars:
+                if field.widget == self.widgets.list.widget and fieldname in request_vars:
                     if fieldname in self.request_vars:
                         value = self.request_vars[fieldname]
                     elif self.record:

gluon/tests/__init__.py

@@ -8,6 +8,7 @@ from .test_dal import *
 from .test_cache import *
 from .test_html import *
 from .test_contribs import *
+from .test_redis import *
 from .test_routes import *
 from .test_router import *
 from .test_authapi import *

gluon/tests/test_redis.py

@@ -0,0 +1,85 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+""" Unit tests for redis """
+
+import unittest
+from datetime import datetime
+
+from gluon._compat import to_bytes, pickle
+from gluon.storage import Storage
+from gluon.utils import web2py_uuid
+from gluon.globals import Request, Response, Session, current
+from gluon.contrib.redis_utils import RConn
+from gluon.contrib.redis_session import RedisSession
+from gluon.contrib.redis_cache import RedisCache
+
+
+class TestRedis(unittest.TestCase):
+    """ Tests the Redis contrib packages """
+    def setUp(self):
+        request = Request(env={})
+        request.application = 'a'
+        request.controller = 'c'
+        request.function = 'f'
+        request.folder = 'applications/admin'
+        response = Response()
+        session = Session()
+        session.connect(request, response)
+        from gluon.globals import current
+        current.request = request
+        current.response = response
+        current.session = session
+        self.current = current
+        rconn = RConn(host='localhost')
+        self.db = RedisSession(redis_conn=rconn, session_expiry=False)
+        self.tname = 'testtablename'
+        return current
+
+    def test_0_redis_session(self):
+        """ Basic redis read-write """
+        db = self.db
+        response = self.current.response
+        Field = db.Field
+        db.define_table(
+            self.tname,
+            Field('locked', 'boolean', default=False),
+            Field('client_ip', length=64),
+            Field('created_datetime', 'datetime',
+                  default=datetime.now().isoformat()),
+            Field('modified_datetime', 'datetime'),
+            Field('unique_key', length=64),
+            Field('session_data', 'blob'),
+        )
+        table = db[self.tname]
+        unique_key = web2py_uuid()
+        dd = dict(
+            locked=0,
+            client_ip=response.session_client,
+            modified_datetime=datetime.now().isoformat(),
+            unique_key=unique_key,
+            session_data=pickle.dumps({'test': 123, 'me': 112312312}, pickle.HIGHEST_PROTOCOL)
+        )
+        record_id = table.insert(**dd)
+        data_from_db = db(table.id == record_id).select()[0]
+        self.assertDictEqual(Storage(dd), data_from_db, 'get inserted dict')
+
+        dd['locked'] = 1
+        table._db(table.id == record_id).update(**dd)
+        data_from_db = db(table.id == record_id).select()[0]
+        self.assertDictEqual(Storage(dd), data_from_db, 'get the updated value')
+
+    def test_1_redis_delete(self):
+        """ Redis session get and delete sessions """
+        db = self.db
+        table = db[self.tname]
+        all_sessions = db(table.id > 0).select()
+        self.assertIsNotNone(all_sessions, 'we must have some keys in db')
+
+        for entry in all_sessions:
+            res = entry.delete_record()
+            self.assertIsNone(res, 'delete should return None')
+
+        empty_sessions = db(table.id > 0).select()
+        self.assertEqual(empty_sessions, [], 'no sessions left')
+

gluon/tools.py

@@ -3726,7 +3726,7 @@ class Auth(AuthAPI):
         are passed to the constructor of class AuthJWT. Look there for documentation.
         """
         if not self.jwt_handler:
-            raise HTTP(400, "Not authorized")
+            raise HTTP(401, "Not authorized")
         else:
             rtn = self.jwt_handler.jwt_token_manager()
             raise HTTP(200, rtn, cookies=None, **current.response.headers)
@@ -3820,7 +3820,7 @@ class Auth(AuthAPI):
 
     def allows_jwt(self, otherwise=None):
         if not self.jwt_handler:
-            raise HTTP(400, "Not authorized")
+            raise HTTP(401, "Not authorized")
         else:
             return self.jwt_handler.allows_jwt(otherwise=otherwise)
 

scripts/extract_sqlite_models.py

@@ -13,6 +13,12 @@ legacy_db(legacy_db.mytable.id>0).select()
 extract_sqlite_models.py -- Copyright (C) Michele Comitini
 This code is distributed with web2py.
 
+Extended version with support of 
+ - "ID_MYTABLE" as REFERENCE PRIMARY KEY
+ - Inline CREATE TABLE declaration in sqlite DB. 
+Copyright (C) Guillaume DELVIT.
+
+
 The regexp code and the dictionary type map was extended from
 extact_mysql_models.py that comes with web2py. extact_mysql_models.py is Copyright (C) Falko Krause.
 
@@ -58,6 +64,10 @@ def get_foreign_keys(sql_lines):
         hit = re.search(r'FOREIGN\s+KEY\s+\("(\S+)"\)\s+REFERENCES\s+"(\S+)"\s+\("(\S+)"\)', line)
         if hit:
             fks[hit.group(1)] =  hit.groups()[1:]
+        else:
+            hit = re.search(r'ID_(\S+)\s+INTEGER', line)
+            if hit:
+                fks['ID_'+hit.group(1)] =  [hit.group(1), 'ID']
 
     return fks
 
@@ -74,6 +84,8 @@ def sqlite(database_name):
         if 'CREATE' in sql_create_stmnt:  # check if the table exists
             #remove garbage lines from sql statement
             sql_lines = sql_create_stmnt.split('\n')
+            if len(sql_lines) == 1 :
+                sql_lines = re.split('[()\n]|, ',sql_create_stmnt)
             sql_lines = [x for x in sql_lines if not(
                 x.startswith('--') or x.startswith('/*') or x == '')]
             #generate the web2py code from the create statement
@@ -84,6 +96,8 @@ def sqlite(database_name):
                 if re.search('KEY', line) or re.search('PRIMARY', line) or re.search('"ID"', line) or line.startswith(')'):
                     continue
                 hit = re.search(r'\[(\S+)\]\s+(\w+(\(\S+\))?),?( .*)?', line)
+                if not hit:
+                    hit = re.search(r'(\S+)\s(\S+)', line)
                 if hit is not None:
                     name, d_type = hit.group(1), hit.group(2)
                     d_type = re.sub(r'(\w+)\(.*', r'\1', d_type)