Merge remote-tracking branch 'remotes/oficial/master' into patch-2

When running scripts, the path used is relative to the web2py root instead of the full path. This causes differences in migrations when using InDBMigrator - this causes pydal to try to repeat migrations already done.

.travis.yml

@@ -6,7 +6,6 @@ dist: "bionic"
 
 services:
   - mysql
-  - redis-server
 
 python:
   - '2.7'
@@ -24,7 +23,6 @@ install:
 before_script:
   - pip install coverage
   - pip install codecov
-  - pip install redis
 
 before_install:
   - mysql -e 'create database pydal;'

appveyor.yml

@@ -1,8 +1,4 @@
 build: false
-before_build:
-  - choco install redis-64
-  - redis-server --service-install
-  - redis-server --service-start
 
 environment:
   matrix:
@@ -30,7 +26,7 @@ init:
 
 install:
   - python -m ensurepip
-  - pip install codecov redis
+  - pip install codecov
   - git submodule update --init --recursive
   # Check that we have the expected version and architecture for Python
   - "python --version"

gluon/contrib/redis_session.py

@@ -14,7 +14,6 @@ from gluon.storage import Storage
 from gluon.contrib.redis_utils import acquire_lock, release_lock
 from gluon.contrib.redis_utils import register_release_lock
 from gluon._compat import to_native
-from datetime import datetime
 
 logger = logging.getLogger("web2py.session.redis")
 
@@ -66,13 +65,13 @@ class RedisClient(object):
 
     def Field(self, fieldname, type='string', length=None, default=None,
               required=False, requires=None):
-        return fieldname, type
+        return None
 
     def define_table(self, tablename, *fields, **args):
         if not self.tablename:
             self.tablename = MockTable(
                 self, self.r_server, tablename, self.session_expiry,
-                with_lock=self.with_lock, fields=fields)
+                self.with_lock)
         return self.tablename
 
     def __getitem__(self, key):
@@ -86,26 +85,10 @@ class RedisClient(object):
         # this is only called by session2trash.py
         pass
 
-    def convert_dict_string(self, dict_string):
-        fields = self.tablename.fields
-        typed_dict = dict()
-        converters = {
-            'boolean': lambda x: 1 if x.decode() == '1' else 0,
-            'blob': lambda x: x,
-        }
-        for field, ftype in fields:
-            if field not in dict_string:
-                continue
-            if ftype in converters:
-                typed_dict[field] = converters[ftype](dict_string[field])
-            else:
-                typed_dict[field] = dict_string[field].decode()
-        return typed_dict
-
 
 class MockTable(object):
 
-    def __init__(self, db, r_server, tablename, session_expiry, with_lock=False, fields=None):
+    def __init__(self, db, r_server, tablename, session_expiry, with_lock=False):
         # here self.db is the RedisClient instance
         self.db = db
         self.tablename = tablename
@@ -118,7 +101,6 @@ class MockTable(object):
         # remember the session_expiry setting
         self.session_expiry = session_expiry
         self.with_lock = with_lock
-        self.fields = fields if fields is not None else []
 
     def __call__(self, record_id, unique_key=None):
         # Support DAL shortcut query: table(record_id)
@@ -190,11 +172,7 @@ class MockQuery(object):
         self.value = value
         self.op = op
 
-    def __ge__(self, value, op='ge'):
+    def __gt__(self, value, op='ge'):
-        self.value = value
-        self.op = op
-
-    def __gt__(self, value, op='gt'):
         self.value = value
         self.op = op
 
@@ -204,7 +182,7 @@ class MockQuery(object):
             key = self.keyprefix + ':' + str(self.value)
             if self.with_lock:
                 acquire_lock(self.db.r_server, key + ':lock', self.value, 2)
-            rtn = {to_native(k): v for k, v in self.db.r_server.hgetall(key).items()}
+            rtn = {to_native(k.decode): v for k, v in self.db.r_server.hgetall(key).items()}
             if rtn:
                 if self.unique_key:
                     # make sure the id and unique_key are correct
@@ -212,8 +190,8 @@ class MockQuery(object):
                         rtn['update_record'] = self.update  # update record support
                     else:
                         rtn = None
-            return [Storage(self.db.convert_dict_string(rtn))] if rtn else []
+            return [Storage(rtn)] if rtn else []
-        elif self.op in ('ge', 'gt') and self.field == 'id' and self.value == 0:
+        elif self.op == 'ge' and self.field == 'id' and self.value == 0:
             # means that someone wants the complete list
             rtn = []
             id_idx = "%s:id_idx" % self.keyprefix
@@ -226,7 +204,7 @@ class MockQuery(object):
                         # clean up the idx, because the key expired
                         self.db.r_server.srem(id_idx, sess)
                     continue
-                val = Storage(self.db.convert_dict_string(val))
+                val = Storage(val)
                 # add a delete_record method (necessary for sessions2trash.py)
                 val.delete_record = RecordDeleter(
                     self.db, sess, self.keyprefix)

gluon/shell.py

@@ -234,7 +234,7 @@ def run(
     errmsg = 'invalid application name: %s' % appname
     if not a:
         die(errmsg, error_preamble=False)
-    adir = os.path.join('applications', a)
+    adir = os.path.absdir(os.path.join('applications', a))
 
     if not os.path.exists(adir):
         if not cron_job and not scheduler_job and \
@@ -273,7 +273,7 @@ def run(
     if vars:
         # underscore necessary because request.vars is a property
         extra_request['_vars'] = vars
-    _env = env(a, c=c, f=f, import_models=import_models, extra_request=extra_request)
+    _env = env(a, c=c, f=f, import_models=import_models, extra_request=extra_request, dir=adir)
 
     if c:
         pyfile = os.path.join('applications', a, 'controllers', c + '.py')

gluon/sqlhtml.py

@@ -461,12 +461,7 @@ class RadioWidget(OptionsWidget):
             opts.append(child(tds))
 
         if opts:
-            opts.append(
+            opts[-1][0][0]['hideerror'] = False
-                INPUT(requires=attr.get('requires', None),
-                      _style="display:none;",
-                      _disabled="disabled",
-                      _name=field.name,
-                      hideerror=False))
         return parent(*opts, **attr)
 
 
@@ -1819,7 +1814,7 @@ class SQLFORM(FORM):
                 if not field.widget and field.type.startswith('list:') and \
                         not OptionsWidget.has_options(field):
                     field.widget = self.widgets.list.widget
-                if field.widget == self.widgets.list.widget and fieldname in request_vars:
+                if field.widget and fieldname in request_vars:
                     if fieldname in self.request_vars:
                         value = self.request_vars[fieldname]
                     elif self.record:

gluon/tests/__init__.py

@@ -8,7 +8,6 @@ from .test_dal import *
 from .test_cache import *
 from .test_html import *
 from .test_contribs import *
-from .test_redis import *
 from .test_routes import *
 from .test_router import *
 from .test_authapi import *

gluon/tests/test_redis.py

@@ -1,85 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-""" Unit tests for redis """
-
-import unittest
-from datetime import datetime
-
-from gluon._compat import to_bytes, pickle
-from gluon.storage import Storage
-from gluon.utils import web2py_uuid
-from gluon.globals import Request, Response, Session, current
-from gluon.contrib.redis_utils import RConn
-from gluon.contrib.redis_session import RedisSession
-from gluon.contrib.redis_cache import RedisCache
-
-
-class TestRedis(unittest.TestCase):
-    """ Tests the Redis contrib packages """
-    def setUp(self):
-        request = Request(env={})
-        request.application = 'a'
-        request.controller = 'c'
-        request.function = 'f'
-        request.folder = 'applications/admin'
-        response = Response()
-        session = Session()
-        session.connect(request, response)
-        from gluon.globals import current
-        current.request = request
-        current.response = response
-        current.session = session
-        self.current = current
-        rconn = RConn(host='localhost')
-        self.db = RedisSession(redis_conn=rconn, session_expiry=False)
-        self.tname = 'testtablename'
-        return current
-
-    def test_0_redis_session(self):
-        """ Basic redis read-write """
-        db = self.db
-        response = self.current.response
-        Field = db.Field
-        db.define_table(
-            self.tname,
-            Field('locked', 'boolean', default=False),
-            Field('client_ip', length=64),
-            Field('created_datetime', 'datetime',
-                  default=datetime.now().isoformat()),
-            Field('modified_datetime', 'datetime'),
-            Field('unique_key', length=64),
-            Field('session_data', 'blob'),
-        )
-        table = db[self.tname]
-        unique_key = web2py_uuid()
-        dd = dict(
-            locked=0,
-            client_ip=response.session_client,
-            modified_datetime=datetime.now().isoformat(),
-            unique_key=unique_key,
-            session_data=pickle.dumps({'test': 123, 'me': 112312312}, pickle.HIGHEST_PROTOCOL)
-        )
-        record_id = table.insert(**dd)
-        data_from_db = db(table.id == record_id).select()[0]
-        self.assertDictEqual(Storage(dd), data_from_db, 'get inserted dict')
-
-        dd['locked'] = 1
-        table._db(table.id == record_id).update(**dd)
-        data_from_db = db(table.id == record_id).select()[0]
-        self.assertDictEqual(Storage(dd), data_from_db, 'get the updated value')
-
-    def test_1_redis_delete(self):
-        """ Redis session get and delete sessions """
-        db = self.db
-        table = db[self.tname]
-        all_sessions = db(table.id > 0).select()
-        self.assertIsNotNone(all_sessions, 'we must have some keys in db')
-
-        for entry in all_sessions:
-            res = entry.delete_record()
-            self.assertIsNone(res, 'delete should return None')
-
-        empty_sessions = db(table.id > 0).select()
-        self.assertEqual(empty_sessions, [], 'no sessions left')
-

gluon/tools.py

@@ -3726,7 +3726,7 @@ class Auth(AuthAPI):
         are passed to the constructor of class AuthJWT. Look there for documentation.
         """
         if not self.jwt_handler:
-            raise HTTP(401, "Not authorized")
+            raise HTTP(400, "Not authorized")
         else:
             rtn = self.jwt_handler.jwt_token_manager()
             raise HTTP(200, rtn, cookies=None, **current.response.headers)
@@ -3820,7 +3820,7 @@ class Auth(AuthAPI):
 
     def allows_jwt(self, otherwise=None):
         if not self.jwt_handler:
-            raise HTTP(401, "Not authorized")
+            raise HTTP(400, "Not authorized")
         else:
             return self.jwt_handler.allows_jwt(otherwise=otherwise)
 

scripts/extract_sqlite_models.py

@@ -13,12 +13,6 @@ legacy_db(legacy_db.mytable.id>0).select()
 extract_sqlite_models.py -- Copyright (C) Michele Comitini
 This code is distributed with web2py.
 
-Extended version with support of 
- - "ID_MYTABLE" as REFERENCE PRIMARY KEY
- - Inline CREATE TABLE declaration in sqlite DB. 
-Copyright (C) Guillaume DELVIT.
-
-
 The regexp code and the dictionary type map was extended from
 extact_mysql_models.py that comes with web2py. extact_mysql_models.py is Copyright (C) Falko Krause.
 
@@ -64,10 +58,6 @@ def get_foreign_keys(sql_lines):
         hit = re.search(r'FOREIGN\s+KEY\s+\("(\S+)"\)\s+REFERENCES\s+"(\S+)"\s+\("(\S+)"\)', line)
         if hit:
             fks[hit.group(1)] =  hit.groups()[1:]
-        else:
-            hit = re.search(r'ID_(\S+)\s+INTEGER', line)
-            if hit:
-                fks['ID_'+hit.group(1)] =  [hit.group(1), 'ID']
 
     return fks
 
@@ -84,8 +74,6 @@ def sqlite(database_name):
         if 'CREATE' in sql_create_stmnt:  # check if the table exists
             #remove garbage lines from sql statement
             sql_lines = sql_create_stmnt.split('\n')
-            if len(sql_lines) == 1 :
-                sql_lines = re.split('[()\n]|, ',sql_create_stmnt)
             sql_lines = [x for x in sql_lines if not(
                 x.startswith('--') or x.startswith('/*') or x == '')]
             #generate the web2py code from the create statement
@@ -96,8 +84,6 @@ def sqlite(database_name):
                 if re.search('KEY', line) or re.search('PRIMARY', line) or re.search('"ID"', line) or line.startswith(')'):
                     continue
                 hit = re.search(r'\[(\S+)\]\s+(\w+(\(\S+\))?),?( .*)?', line)
-                if not hit:
-                    hit = re.search(r'(\S+)\s(\S+)', line)
                 if hit is not None:
                     name, d_type = hit.group(1), hit.group(2)
                     d_type = re.sub(r'(\w+)\(.*', r'\1', d_type)