remove useless imports.

Add support for freeIPA ldap auth

.travis.yml

@@ -1,27 +1,22 @@
 language: python
 
 sudo: required
-
 cache: pip
-
-dist: "xenial"
+dist: "bionic"
 
 services:
   - mysql
+  - redis-server
 
 python:
   - '2.7'
-  - '3.5'
   - '3.6'
-  - '3.6-dev'
   - '3.7'
-  - '3.7-dev'
   - 'pypy3.5'
 
 matrix:
   allow_failures:
-    - python: '3.6-dev'
-    - python: '3.7-dev'
+    - python: 'pypy3.5'
 
 install:
   - pip install -e .
@@ -29,6 +24,7 @@ install:
 before_script:
   - pip install coverage
   - pip install codecov
+  - pip install redis
 
 before_install:
   - mysql -e 'create database pydal;'

applications/admin/static/js/web2py.js

@@ -284,6 +284,7 @@
                 var redirect = xhr.getResponseHeader('web2py-redirect-location');
                 if (redirect !== null) {
                     window.location = redirect;
+                    window.location.reload(); // Force reload even with anchors
                 }
                 /* run this here only if this Ajax request is NOT for a web2py component. */
                 if (xhr.getResponseHeader('web2py-component-content') === null) {

applications/examples/views/default/documentation.html

@@ -3,7 +3,7 @@
 <div>
   {{=get_content('main')}}
   <center>
-    <iframe src="//player.vimeo.com/hubnut/album/3016728?color=ff6600&amp;background=ffffff&amp;slideshow=1&amp;video_title=1&amp;video_byline=1" width="400" height="300" frameborder="0" webkitAllowFullScreen mozallowfullscreen allowFullScreen></iframe>
+    <iframe src="https://player.vimeo.com/video/104800778?color=ff6600&amp;background=ffffff&amp;slideshow=1&amp;video_title=1&amp;video_byline=1" width="400" height="300" frameborder="0" webkitAllowFullScreen mozallowfullscreen allowFullScreen></iframe>
   </center>
   {{=get_content('official')}}
   {{=get_content('community')}}

applications/examples/views/default/download.html

@@ -9,7 +9,8 @@
   <table class="twothirds">
     <thead>
       <tr>
-        <th>For Normal Users</th>
+        <th>For Normal Users (Py3)</th>
+        <th>For Legacy Users (Py2)</th>
         <th>For Testers</th>
         <th>For Developers</th>
       </tr>
@@ -17,10 +18,13 @@
     <tbody>
       <tr>
         <td>
-          <a class="btn btn180 rounded green" href="https://mdipierro.pythonanywhere.com/examples/static/web2py_win.zip">For Windows</a>
+          <a class="btn btn180 rounded green" href="https://mdipierro.pythonanywhere.com/examples/static/web2py_win.zip">Windows binaries</a>
         </td>
         <td>
-          <a class="btn btn180 rounded yellow" href="https://mdipierro.pythonanywhere.com/examples/static/nightly/web2py_win.zip">For Windows</a>
+          <a class="btn btn180 rounded" href="https://mdipierro.pythonanywhere.com/examples/static/web2py_win_py2.zip">Windows binaries</a>
+        </td>
+        <td>
+          <a class="btn btn180 rounded yellow" href="https://mdipierro.pythonanywhere.com/examples/static/nightly/web2py_win.zip">Windows binaries</a>
         </td>
         <td>
           <a class="btn btn180 rounded red" href="http://github.com/web2py/web2py/">Git Repository</a>
@@ -28,17 +32,25 @@
       </tr>
       <tr>
         <td>
-          <a class="btn btn180 rounded green" href="https://mdipierro.pythonanywhere.com/examples/static/web2py_osx.zip">For Mac</a>
+          <a class="btn btn180 rounded green" href="https://mdipierro.pythonanywhere.com/examples/static/web2py_osx.zip">Mac binaries</a>
         </td>
         <td>
-          <a class="btn btn180 rounded yellow" href="https://mdipierro.pythonanywhere.com/examples/static/nightly/web2py_osx.zip">For Mac</a>
+          <a class="btn btn180 rounded" href="https://mdipierro.pythonanywhere.com/examples/static/web2py_osx_py2.zip">Mac binaries</a>
+        </td>
+        <td>
+          <a class="btn btn180 rounded yellow" href="https://mdipierro.pythonanywhere.com/examples/static/nightly/web2py_osx.zip">Mac binaries</a>
+        </td>
+        <td>
+          <a class="btn btn180 rounded" href="http://mdipierro.github.io/web2py/web2py_manual_5th.pdf">Manual</a>
         </td>
-        <td></td>
       </tr>
       <tr>
         <td>
           <a class="btn btn180 rounded green" href="https://mdipierro.pythonanywhere.com/examples/static/web2py_src.zip">Source Code</a>
         </td>
+        <td>
+          <a class="btn btn180 rounded" href="https://mdipierro.pythonanywhere.com/examples/static/web2py_src.zip">Source Code</a>
+        </td>
         <td>
           <a class="btn btn180 rounded yellow" href="https://mdipierro.pythonanywhere.com/examples/static/nightly/web2py_src.zip">Source Code</a>
         </td>
@@ -48,7 +60,8 @@
       </tr>
       <tr>
         <td>
-          <a class="btn btn180 rounded green" href="http://mdipierro.github.io/web2py/web2py_manual_5th.pdf">Manual</a>
+        </td>
+        <td>
         </td>
         <td>
           <a class="btn btn180 rounded" href="https://github.com/web2py/web2py/releases">Change Log</a>
@@ -62,13 +75,19 @@
 </center>
 
 <p style="text-align:left;">
-  The source code version works on Windows and most Unix systems, including <b>Linux</b>,  <b>BSD</b> and  <b>Mac</b> . It requires Python 2.6 (no more supported),  Python 2.7 (stable) or Python 3.5+ (recommended for new projects) already installed on your system.
-  There are also binary packages for Windows and Mac OS X. They include the Python 2.7 interpreter so you do not need to have it pre-installed.
+  The source code version works on Windows and most Unix systems, including <b>Linux</b>,  <b>BSD</b> and  <b>Mac</b> . It requires Python 3.5+ (recommended for new projects) 
+  or Python 2.7+ (stable, for use with legacy apps) already installed on your system.
+</p>
+<p style="text-align:left;">
+  There are also binary packages for Windows and MacOs. They include the Python interpreter version 3.7.4 or 2.7.16, so you do not need to have it pre-installed.
 </p>
 
 <h3>Instructions</h3>
-<p>With the binary packages, after download, just unzip it and then click on web2py.exe (windows) or web2py.app (osx).
-  If you prefer to run it from source with your own Python interpreter alreay installed, type:</p>
+<p>With the binary packages, after download, just unzip it and then click on web2py.exe (Windows) or web2py (MacOs).</p>
+<p>Note that on recent MacOs versions (10.12+) you could face problems in running the binary App program, due to the last changes to the security settings. 
+In this case, press the 'control' key + click on downloaded file and then 'open' it (confirm the warnings). Finally move the program in Applications and run it from there.
+</p>
+<p> If you prefer to run it from source with your own Python interpreter already installed, type:</p>
 {{=CODE("python web2py.py", language=None, counter='>', _class='boxCode')}}
 <p>or for more info type:</p>
 {{=CODE("python web2py.py -h", language=None, counter='>', _class='boxCode')}}

applications/welcome/static/js/web2py.js

@@ -283,7 +283,11 @@
             doc.ajaxSuccess(function (e, xhr) {
                 var redirect = xhr.getResponseHeader('web2py-redirect-location');
                 if (redirect !== null) {
-                    window.location = redirect;
+                    if (!redirect.endsWith('#')) {
+                        window.location.href = redirect;
+                    } else {
+                        window.location.reload();
+                    }
                 }
                 /* run this here only if this Ajax request is NOT for a web2py component. */
                 if (xhr.getResponseHeader('web2py-component-content') === null) {
@@ -335,7 +339,7 @@
                     } else {
                         formData = form.serialize(); // Fallback for older browsers.
                     }
-                    web2py.ajax_page('post', url, formData, target, form);
+                    web2py.ajax_page('post', url, formData, target);
 
                     e.preventDefault();
                 });
@@ -367,6 +371,24 @@
                     'data': data,
                     'processData': !isFormData,
                     'contentType': contentType,
+                    'xhr': function() {
+                        var xhr = new window.XMLHttpRequest();
+
+                        xhr.upload.addEventListener("progress", function(evt) {
+                            if (evt.lengthComputable) {
+                                var percentComplete = evt.loaded / evt.total;
+                                percentComplete = parseInt(percentComplete * 100);
+                                web2py.fire(element, 'w2p:uploadProgress', [percentComplete], target);
+
+                                if (percentComplete === 100) {
+                                    web2py.fire(element, 'w2p:uploadComplete', [], target);
+                                }
+
+                            }
+                        }, false);
+
+                        return xhr;
+                    },
                     'beforeSend': function (xhr, settings) {
                         xhr.setRequestHeader('web2py-component-location', document.location);
                         xhr.setRequestHeader('web2py-component-element', target);

appveyor.yml

@@ -1,4 +1,8 @@
 build: false
+before_build:
+  - choco install redis-64
+  - redis-server --service-install
+  - redis-server --service-start
 
 environment:
   matrix:
@@ -26,7 +30,7 @@ init:
 
 install:
   - python -m ensurepip
-  - pip install codecov
+  - pip install codecov redis
   - git submodule update --init --recursive
   # Check that we have the expected version and architecture for Python
   - "python --version"

extras/build_web2py/README_mac.md

@@ -0,0 +1,50 @@
+## MacOS binaries
+
+The MacOS binaries contain Python 3.7.3 (or 2.7.16) 64 bit with all the needed modules and the web2py in the specified version: 
+ you don't need anything else to run them on MacOS! After uncompressing the zip file, you just need to click on the web2py icon inside.  
+
+They were produced on MacOS Sierra 10.12.6 + security update 2019.001.
+
+## Full MacOS build recipe
+
+1. grab and install the official Python program: we've got version 3.7.3 or 2.7.16 (64 bit). If you've chosen python 2, change pip3
+with pip, and python3 with python in the following instructions...
+
+2. Open a terminal, update tools with:
+
+"python3 -m pip install --upgrade pip"  
+"pip3 install --upgrade setuptools"   
+
+
+3. install PyInstaller with:  
+sudo -H pip3 install pyinstaller (we've got PyInstaller-3.4 )
+
+4. additional (but not required) packages:  
+(only for python 2: install Homebrew from https://brew.sh/#install , then 'brew install unixodbc' )  
+pip3 install psycopg2-binary = psycopg2-2.7.7  
+pip3 install pyodbc = pyodbc-4.0.26-cp37-cp37m  
+pip3 install python-ldap (on the windows message, accept to install the "Command line developer tools"). Rerun:  
+pip3 install python-ldap  
+
+5. grab latest web2py source from https://mdipierro.pythonanywhere.com/examples/static/web2py_src.zip 
+ (you need at least 2.18.3 for needed changes in gluon\admin.py). Open it to uncompress, in this example on Desktop/web2py
+
+
+6. take the file build_web2py.py and web2py.mac.spec  from this folder and place it on the Desktop/web2py  folder  
+
+7. edit the file /Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/PyInstaller/hooks/hook-_tkinter.py 
+ and change one of its line according to https://github.com/pyinstaller/pyinstaller/pull/3830  
+
+8. (optional, for having a full working interactive shell) change the fake site.py module included within the PyInstaller installation 
+ with the content of the files web2py.site_37.py or web2py.site_27.py from this folder - see comments inside these files for details
+
+9. open a terminal, goto Desktop/web2py and run:  
+  
+python3 build_web2py.py
+
+10. if everything is fine, you'll obtain web2py_macos.zip on the Desktop/web2py  folder. Inside it, there is the web2py program with 
+ both the CMD version and the APP version.
+
+## Gothca
+
+Unfortunately, the APP version is still not working - see https://github.com/pyinstaller/pyinstaller/issues/3820 . 

extras/build_web2py/README_win.md

@@ -0,0 +1,47 @@
+## Windows binaries
+
+The windows binaries contain Python 64 bit version 3.7.3 or 2.7.16 with all the needed modules and the web2py in the specified version.
+You don't need anything else to run them on Windows.  
+At least on Windows 7, if you get an error stating that "api-ms-win-crt-runtime-l1-1-0.dll is missing" you have only to install the
+free and official "Visual C++ Redistributable for Visual Studio" as described later
+
+
+## Full Windows build recipe
+
+1. get a clean Windows 10 (Windows 10 Professional English build 1809 64 bit, under Virtualbox in our case)
+2. grab and install the official Python program: we've got version 3.7.3 or 2.7.16, 64 bit 
+(https://www.python.org/ftp/python/3.7.2/python-3.7.2-amd64.exe ) + select  "add Python 3.7 to PATH" during its setup if Python 3. 
+For Python 2 you need to manually add the folders for python27 and python27\Scripts to the system path.
+3. update tools with   
+"python -m pip install --upgrade pip"  
+"pip install --upgrade setuptools"  
+4. download and install python-win32, which is needed for web2py to work with all features enabled 
+(https://github.com/mhammond/pywin32/releases/download/b224/pywin32-224.win-amd64-py3.7.exe)
+5. grab latest web2py source from https://mdipierro.pythonanywhere.com/examples/static/web2py_src.zip (you need at least 2.18.3 for 
+needed changes in gluon\admin.py). Unzip it in a dedicated folder, in this example C:\web2py - so that you have 
+C:\web2py\web2py.py inside)
+6. install PyInstaller with:  
+        pip install pyinstaller  (we've got PyInstaller-3.4.tar.gz )  
+7. download and install the free Microsoft Visual C++ Redistributable per Visual Studio 2017, 64 bit version, from 
+ https://aka.ms/vs/15/release/vc_redist.x64.exe  
+8. additional (but not required) packages to work better in the Windows world:  
+pip install psycopg2 = psycopg2-2.7.7-cp37-cp37m-win_amd64.whl  
+pip install pyodbc = pyodbc-4.0.26-cp37-cp37m-win_amd64.whl  
+download the file python_ldap-3.1.0-cp37-cp37m-win_amd64.whl from https://www.lfd.uci.edu/~gohlke/pythonlibs/ and install it from that 
+folder with the command 'pip install python_ldap-3.1.0-cp37-cp37m-win_amd64.whl'  
+
+9. copy build_web2py.py, web2py.win.spec and web2py.win_no_console.spec from this folder to C:\web2py\  
+10. (only for python 2) - due to a PyInstaller bug, you need to manually change the file gluon\rocket.py, line 26, from IS_JYTHON 
+= platform.system() == 'Java'  to  IS_JYTHON = False
+11. (optional, for having a full working interactive shell) change the fake site.py module included within the PyInstaller installation
+with the content of the files web2py.site_37.py or web2py.site_27.py from this folder - see comments inside these files for details 
+12. open a CMD and go to C:\web2py. Run:
+
+    python build_web2py.py
+
+If everything goes fine, you'll obtain the 64 bit binary build zipped as C:\web2py\web2py_win.zip.
+If you try to run it in a 32 bit Windows system, you'll correctly get a 'web2py.exe not a valid Win32 application' error message.
+
+## Gothca:
+- at least on Windows 7, you can get an error stating that "api-ms-win-crt-runtime-l1-1-0.dll is missing". You can easily resolve it by
+installing "Visual C++ Redistributable for Visual Studio" described earlier

gluon/authapi.py

@@ -803,7 +803,7 @@ class AuthAPI(object):
 
         # Finally verify the password
         passfield = settings.password_field
-        password = table_user[passfield].validate(kwargs.get(passfield, ''))[0]
+        password = table_user[passfield].validate(kwargs.get(passfield, ''), None)[0]
 
         if password == user[passfield]:
             self.login_user(user)

gluon/compileapp.py

@@ -565,7 +565,7 @@ def run_models_in(environment):
 
 TEST_CODE = r"""
 def _TEST():
-    import doctest, sys, cStringIO, types, cgi, gluon.fileutils
+    import doctest, sys, cStringIO, types, gluon.fileutils
     if not gluon.fileutils.check_credentials(request):
         raise HTTP(401, web2py_error='invalid credentials')
     stdout = sys.stdout

gluon/contrib/login_methods/freeipa_auth.py

@@ -0,0 +1,59 @@
+import sys
+import logging
+
+try:
+    import ldap
+
+    ldap.set_option(ldap.OPT_X_TLS_REQUIRE_CERT, ldap.OPT_X_TLS_NEVER)
+except Exception as detail:
+    logging.error('missing ldap, try "pip install python-ldap"')
+    raise detail
+
+
+def freeipa_auth(server, basedn, group):
+    """
+    custom module for freeIPA auth in web2py
+    server: freeipa ip
+    base_dn: root of ldap tree containing user & groups
+    group: group authing user has to be a member of
+
+   """
+    logger = logging.getLogger("web2py.auth.freeipa_auth")
+
+    def freeipa_auth_aux(username, password):
+        if password == "" or username == "":
+            logger.warning("blank username / password not allowed")
+            return False
+
+        bind_user_base = "uid=" + username + ",cn=users," + basedn
+        ldap_filter = "memberof=cn=" + group + ",cn=groups," + basedn
+
+        session = ldap.initialize("ldaps://" + server + ":636")
+        try:
+            session.bind_s(bind_user_base, password)
+        except ldap.LDAPError:
+            import traceback
+
+            logger.warning("[%s] Error in ldap bind" % str(username))
+            logger.debug(traceback.format_exc())
+            return False
+
+        try:
+            result = session.search_s(
+                bind_user_base, ldap.SCOPE_SUBTREE, ldap_filter, ["member"]
+            )
+            session.unbind()
+        except ldap.LDAPError as detail:
+            logger.warning(
+                "ldap_auth: searc %s for %s resulted in %s: %s\n"
+                % (bind_user_base, ldap_filter, exc_type, exc_value)
+            )
+
+        try:
+            if result == list():
+                return False
+            return True
+        except:
+            return False
+
+    return freeipa_auth_aux

gluon/contrib/login_methods/saml2_auth.py

@@ -150,14 +150,14 @@ class Saml2Auth(object):
         self.config_file = config_file
         self.maps = maps
 
-    # URL for redirecting users to when they sign out
+        # URL for redirecting users to when they sign out
         self.saml_logout_url = logout_url
 
         # URL to let users change their password in the IDP system
         self.saml_change_password_url = change_password_url
     
-    # URL to specify an IDP if using federation metadata or an MDQ
-    self.entityid = entityid
+        # URL to specify an IDP if using federation metadata or an MDQ
+        self.entityid = entityid
 
     def login_url(self, next="/"):
         d = saml2_handler(current.session, current.request, entityid=self.entityid)

gluon/contrib/markmin/markmin2latex.py

@@ -237,6 +237,7 @@ WRAPPER = """
 \\usepackage{graphicx}
 \\usepackage{grffile}
 \\usepackage[utf8x]{inputenc}
+\\usepackage{textgreek}
 \\definecolor{lg}{rgb}{0.9,0.9,0.9}
 \\definecolor{dg}{rgb}{0.3,0.3,0.3}
 \\def\\ft{\\small\\tt}

gluon/contrib/markmin/markmin2pdf.py

@@ -80,7 +80,7 @@ def latex2pdf(latex, pdflatex='pdflatex', passes=3):
             outfile.close()
         re_errors = re.compile('^\!(.*)$', re.M)
         re_warnings = re.compile('^LaTeX Warning\:(.*)$', re.M)
-        flog = open(logname)
+        flog = open(logname,encoding="utf-8")
         try:
             loglines = flog.read()
         finally:

gluon/contrib/redis_cache.py

@@ -208,12 +208,12 @@ class RedisClient(object):
 
     def retry_call(self, key, f, time_expire, with_lock):
         self.RETRIES += 1
-        if self.RETRIES <= self.MAX_RETRIES:
-            logger.error("sleeping %s seconds before reconnecting" % (2 * self.RETRIES))
-            time.sleep(2 * self.RETRIES)
+        if self.RETRIES <= self.MAX_RETRIES:            
             if self.fail_gracefully:
                 self.RETRIES = 0
                 return f()
+            logger.error("sleeping %s seconds before reconnecting" % (2 * self.RETRIES))            
+            time.sleep(2 * self.RETRIES)
             return self.__call__(key, f, time_expire, with_lock)
         else:
             self.RETRIES = 0

gluon/contrib/redis_session.py

@@ -13,7 +13,8 @@ from gluon import current
 from gluon.storage import Storage
 from gluon.contrib.redis_utils import acquire_lock, release_lock
 from gluon.contrib.redis_utils import register_release_lock
-from gluon._compat import to_bytes
+from gluon._compat import to_native
+from datetime import datetime
 
 logger = logging.getLogger("web2py.session.redis")
 
@@ -65,13 +66,13 @@ class RedisClient(object):
 
     def Field(self, fieldname, type='string', length=None, default=None,
               required=False, requires=None):
-        return None
+        return fieldname, type
 
     def define_table(self, tablename, *fields, **args):
         if not self.tablename:
             self.tablename = MockTable(
                 self, self.r_server, tablename, self.session_expiry,
-                self.with_lock)
+                with_lock=self.with_lock, fields=fields)
         return self.tablename
 
     def __getitem__(self, key):
@@ -85,10 +86,26 @@ class RedisClient(object):
         # this is only called by session2trash.py
         pass
 
+    def convert_dict_string(self, dict_string):
+        fields = self.tablename.fields
+        typed_dict = dict()
+        converters = {
+            'boolean': lambda x: 1 if x.decode() == '1' else 0,
+            'blob': lambda x: x,
+        }
+        for field, ftype in fields:
+            if field not in dict_string:
+                continue
+            if ftype in converters:
+                typed_dict[field] = converters[ftype](dict_string[field])
+            else:
+                typed_dict[field] = dict_string[field].decode()
+        return typed_dict
+
 
 class MockTable(object):
 
-    def __init__(self, db, r_server, tablename, session_expiry, with_lock=False):
+    def __init__(self, db, r_server, tablename, session_expiry, with_lock=False, fields=None):
         # here self.db is the RedisClient instance
         self.db = db
         self.tablename = tablename
@@ -101,6 +118,7 @@ class MockTable(object):
         # remember the session_expiry setting
         self.session_expiry = session_expiry
         self.with_lock = with_lock
+        self.fields = fields if fields is not None else []
 
     def __call__(self, record_id, unique_key=None):
         # Support DAL shortcut query: table(record_id)
@@ -172,7 +190,11 @@ class MockQuery(object):
         self.value = value
         self.op = op
 
-    def __gt__(self, value, op='ge'):
+    def __ge__(self, value, op='ge'):
+        self.value = value
+        self.op = op
+
+    def __gt__(self, value, op='gt'):
         self.value = value
         self.op = op
 
@@ -182,16 +204,16 @@ class MockQuery(object):
             key = self.keyprefix + ':' + str(self.value)
             if self.with_lock:
                 acquire_lock(self.db.r_server, key + ':lock', self.value, 2)
-            rtn = self.db.r_server.hgetall(key)
+            rtn = {to_native(k): v for k, v in self.db.r_server.hgetall(key).items()}
             if rtn:
                 if self.unique_key:
                     # make sure the id and unique_key are correct
-                    if rtn['unique_key'] == self.unique_key:
+                    if rtn['unique_key'] == to_native(self.unique_key):
                         rtn['update_record'] = self.update  # update record support
                     else:
                         rtn = None
-            return [Storage(rtn)] if rtn else []
-        elif self.op == 'ge' and self.field == 'id' and self.value == 0:
+            return [Storage(self.db.convert_dict_string(rtn))] if rtn else []
+        elif self.op in ('ge', 'gt') and self.field == 'id' and self.value == 0:
             # means that someone wants the complete list
             rtn = []
             id_idx = "%s:id_idx" % self.keyprefix
@@ -204,7 +226,7 @@ class MockQuery(object):
                         # clean up the idx, because the key expired
                         self.db.r_server.srem(id_idx, sess)
                     continue
-                val = Storage(val)
+                val = Storage(self.db.convert_dict_string(val))
                 # add a delete_record method (necessary for sessions2trash.py)
                 val.delete_record = RecordDeleter(
                     self.db, sess, self.keyprefix)

gluon/contrib/simplejsonrpc.py

@@ -30,7 +30,7 @@ else:
     from io import StringIO
 import random
 import json
-
+from gluon._compat import basestring
 
 class JSONRPCError(RuntimeError):
     "Error object for remote procedure call fail"

gluon/globals.py

@@ -13,7 +13,7 @@ Contains the classes for the global used variables:
 
 """
 from gluon._compat import pickle, StringIO, copyreg, Cookie, urlparse, PY2, iteritems, to_unicode, to_native, \
-    to_bytes, unicodeT, long, hashlib_md5, urllib_quote
+    to_bytes, unicodeT, long, hashlib_md5, urllib_quote, to_native
 from gluon.storage import Storage, List
 from gluon.streamer import streamer, stream_file_or_304_or_206, DEFAULT_CHUNK_SIZE
 from gluon.contenttype import contenttype
@@ -1055,7 +1055,7 @@ class Session(Storage):
             if record_id.isdigit() and long(record_id) > 0:
                 new_unique_key = web2py_uuid()
                 row = table(record_id)
-                if row and row['unique_key'] == unique_key:
+                if row and to_native(row['unique_key']) == to_native(unique_key):
                     table._db(table.id == record_id).update(unique_key=new_unique_key)
                 else:
                     record_id = None
@@ -1231,9 +1231,9 @@ class Session(Storage):
 
         session_pickled = response.session_pickled or pickle.dumps(self, pickle.HIGHEST_PROTOCOL)
 
-        dd = dict(locked=False,
+        dd = dict(locked=0,
                   client_ip=response.session_client,
-                  modified_datetime=request.now,
+                  modified_datetime=request.now.isoformat(),
                   session_data=session_pickled,
                   unique_key=unique_key)
         if record_id:

gluon/languages.py

@@ -16,7 +16,6 @@ import re
 import sys
 import pkgutil
 import logging
-from cgi import escape
 from threading import RLock
 
 from pydal._compat import copyreg, PY2, maketrans, iterkeys, unicodeT, to_unicode, to_bytes, iteritems, to_native, pjoin

gluon/rewrite.py

@@ -204,7 +204,7 @@ def url_out(request, environ, application, controller, function,
     if host is True or (host is None and (scheme or port is not None)):
         host = request.env.http_host
     if not scheme or scheme is True:
-        scheme = 'https' if request and request.is_https else 'http'
+        scheme = request.env.get('wsgi_url_scheme', 'http').lower() if request else 'http'
     if host:
         host_port = host if not port else host.split(':', 1)[0] + ':%s' % port
         url = '%s://%s%s' % (scheme, host_port, url)

gluon/scheduler.py

@@ -530,7 +530,7 @@ class IS_CRONLINE(object):
     def __init__(self, error_message=None):
         self.error_message = error_message
 
-    def __call__(self, value):
+    def __call__(self, value, record_id=None):
         recur = CronParser(value, datetime.datetime.now())
         try:
             recur.next()
@@ -550,7 +550,7 @@ class TYPE(object):
         self.myclass = myclass
         self.parse = parse
 
-    def __call__(self, value):
+    def __call__(self, value, record_id=None):
         from gluon import current
         try:
             obj = loads(value)
@@ -596,12 +596,12 @@ class Scheduler(threading.Thread):
         utc_time(bool): do all datetime calculations assuming UTC as the
             timezone. Remember to pass `start_time` and `stop_time` to tasks
             accordingly
-
+        use_spawn(bool): use spawn for subprocess (only useable with python3)
     """
 
     def __init__(self, db, tasks=None, migrate=True,
                  worker_name=None, group_names=None, heartbeat=HEARTBEAT,
-                 max_empty_runs=0, discard_results=False, utc_time=False):
+                 max_empty_runs=0, discard_results=False, utc_time=False, use_spawn=False):
 
         threading.Thread.__init__(self)
         self.setDaemon(True)
@@ -639,6 +639,7 @@ class Scheduler(threading.Thread):
         current._scheduler = self
 
         self.define_tables(db, migrate=migrate)
+        self.use_spawn = use_spawn
 
     def execute(self, task):
         """Start the background process.
@@ -649,10 +650,19 @@ class Scheduler(threading.Thread):
         Returns:
             a `TaskReport` object
         """
-        outq = multiprocessing.Queue()
-        retq = multiprocessing.Queue(maxsize=1)
-        self.process = p = \
-            multiprocessing.Process(target=executor, args=(retq, task, outq))
+        outq = None
+        retq = None
+        if (self.use_spawn and not PY2):
+            ctx = multiprocessing.get_context('spawn')
+            outq = ctx.Queue()
+            retq = ctx.Queue(maxsize=1)
+            sel.process = p = ctx.Process(target=executor, args=(retq, task, outq))
+        else:
+            outq = multiprocessing.Queue()
+            retq = multiprocessing.Queue(maxsize=1)
+            self.process = p = \
+                multiprocessing.Process(target=executor, args=(retq, task, outq))
+
         self.process_queues = (retq, outq)
 
         logger.debug('   task starting')

gluon/serializers.py

@@ -82,7 +82,7 @@ def custom_json(o):
     elif isinstance(o, integer_types):
         return int(o)
     elif isinstance(o, decimal.Decimal):
-        return str(o)
+        return float(o)
     elif isinstance(o, (bytes, bytearray)):
         return str(o)
     elif isinstance(o, lazyT):

gluon/tests/__init__.py

@@ -8,6 +8,7 @@ from .test_dal import *
 from .test_cache import *
 from .test_html import *
 from .test_contribs import *
+from .test_redis import *
 from .test_routes import *
 from .test_router import *
 from .test_authapi import *

gluon/tests/test_redis.py

@@ -0,0 +1,85 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+""" Unit tests for redis """
+
+import unittest
+from datetime import datetime
+
+from gluon._compat import to_bytes, pickle
+from gluon.storage import Storage
+from gluon.utils import web2py_uuid
+from gluon.globals import Request, Response, Session, current
+from gluon.contrib.redis_utils import RConn
+from gluon.contrib.redis_session import RedisSession
+from gluon.contrib.redis_cache import RedisCache
+
+
+class TestRedis(unittest.TestCase):
+    """ Tests the Redis contrib packages """
+    def setUp(self):
+        request = Request(env={})
+        request.application = 'a'
+        request.controller = 'c'
+        request.function = 'f'
+        request.folder = 'applications/admin'
+        response = Response()
+        session = Session()
+        session.connect(request, response)
+        from gluon.globals import current
+        current.request = request
+        current.response = response
+        current.session = session
+        self.current = current
+        rconn = RConn(host='localhost')
+        self.db = RedisSession(redis_conn=rconn, session_expiry=False)
+        self.tname = 'testtablename'
+        return current
+
+    def test_0_redis_session(self):
+        """ Basic redis read-write """
+        db = self.db
+        response = self.current.response
+        Field = db.Field
+        db.define_table(
+            self.tname,
+            Field('locked', 'boolean', default=False),
+            Field('client_ip', length=64),
+            Field('created_datetime', 'datetime',
+                  default=datetime.now().isoformat()),
+            Field('modified_datetime', 'datetime'),
+            Field('unique_key', length=64),
+            Field('session_data', 'blob'),
+        )
+        table = db[self.tname]
+        unique_key = web2py_uuid()
+        dd = dict(
+            locked=0,
+            client_ip=response.session_client,
+            modified_datetime=datetime.now().isoformat(),
+            unique_key=unique_key,
+            session_data=pickle.dumps({'test': 123, 'me': 112312312}, pickle.HIGHEST_PROTOCOL)
+        )
+        record_id = table.insert(**dd)
+        data_from_db = db(table.id == record_id).select()[0]
+        self.assertDictEqual(Storage(dd), data_from_db, 'get inserted dict')
+
+        dd['locked'] = 1
+        table._db(table.id == record_id).update(**dd)
+        data_from_db = db(table.id == record_id).select()[0]
+        self.assertDictEqual(Storage(dd), data_from_db, 'get the updated value')
+
+    def test_1_redis_delete(self):
+        """ Redis session get and delete sessions """
+        db = self.db
+        table = db[self.tname]
+        all_sessions = db(table.id > 0).select()
+        self.assertIsNotNone(all_sessions, 'we must have some keys in db')
+
+        for entry in all_sessions:
+            res = entry.delete_record()
+            self.assertIsNone(res, 'delete should return None')
+
+        empty_sessions = db(table.id > 0).select()
+        self.assertEqual(empty_sessions, [], 'no sessions left')
+

gluon/tests/test_serializers.py

@@ -44,7 +44,7 @@ class TestSerializers(unittest.TestCase):
         # self.assertEqual(json(1), json(1))
         # decimal stringified
         obj = {'a': decimal.Decimal('4.312312312312')}
-        self.assertEqual(json(obj), u'{"a": "4.312312312312"}')
+        self.assertEqual(json(obj), u'{"a": 4.312312312312}')
         # lazyT translated
         T = TranslatorFactory('', 'en')
         lazy_translation = T('abc')

gluon/tools.py

@@ -2301,7 +2301,7 @@ class Auth(AuthAPI):
         # in this case they will have to reset their password to login
         if fields.get(settings.passfield):
             fields[settings.passfield] = \
-                settings.table_user[settings.passfield].validate(fields[settings.passfield])[0]
+                settings.table_user[settings.passfield].validate(fields[settings.passfield], None)[0]
         if not fields.get(settings.userfield):
             raise ValueError('register_bare: userfield not provided or invalid')
         user = self.get_or_create_user(fields, login=False, get=False,
@@ -2638,9 +2638,7 @@ class Auth(AuthAPI):
                         # invalid login
                         session.flash = specific_error if self.settings.login_specify_error else self.messages.invalid_login
                         callback(onfail, None)
-                        if 'password' in request.post_vars:
-                            del request.post_vars['password']
-                        redirect(self.url(args=request.args, vars=request.vars),client_side=settings.client_side)
+                        redirect(self.url(args=request.args, vars=request.get_vars),client_side=settings.client_side)
 
             else:  # use a central authentication server
                 cas = settings.login_form
@@ -3173,12 +3171,12 @@ class Auth(AuthAPI):
                         formname='retrieve_password', dbio=False,
                         onvalidation=onvalidation, hideerror=self.settings.hideerror):
             user = table_user(email=form.vars.email)
-            key = user.registration_key
             if not user:
                 current.session.flash = \
                     self.messages.invalid_email
                 redirect(self.url(args=request.args))
-            elif key in ('pending', 'disabled', 'blocked') or (key or '').startswith('pending'):
+            key = user.registration_key
+            if key in ('pending', 'disabled', 'blocked') or (key or '').startswith('pending'):
                 current.session.flash = \
                     self.messages.registration_pending
                 redirect(self.url(args=request.args))