Merge remote-tracking branch 'remotes/oficial/master' into patch-2

Add support for freeIPA ldap auth

.travis.yml

@@ -1,27 +1,21 @@
 language: python
 
 sudo: required
-
 cache: pip
-
-dist: "xenial"
+dist: "bionic"
 
 services:
   - mysql
 
 python:
   - '2.7'
-  - '3.5'
   - '3.6'
-  - '3.6-dev'
   - '3.7'
-  - '3.7-dev'
   - 'pypy3.5'
 
 matrix:
   allow_failures:
-    - python: '3.6-dev'
-    - python: '3.7-dev'
+    - python: 'pypy3.5'
 
 install:
   - pip install -e .

applications/examples/views/default/documentation.html

@@ -3,7 +3,7 @@
 <div>
   {{=get_content('main')}}
   <center>
-    <iframe src="//player.vimeo.com/hubnut/album/3016728?color=ff6600&amp;background=ffffff&amp;slideshow=1&amp;video_title=1&amp;video_byline=1" width="400" height="300" frameborder="0" webkitAllowFullScreen mozallowfullscreen allowFullScreen></iframe>
+    <iframe src="https://player.vimeo.com/video/104800778?color=ff6600&amp;background=ffffff&amp;slideshow=1&amp;video_title=1&amp;video_byline=1" width="400" height="300" frameborder="0" webkitAllowFullScreen mozallowfullscreen allowFullScreen></iframe>
   </center>
   {{=get_content('official')}}
   {{=get_content('community')}}

applications/examples/views/default/download.html

@@ -9,7 +9,8 @@
   <table class="twothirds">
     <thead>
       <tr>
-        <th>For Normal Users</th>
+        <th>For Normal Users (Py3)</th>
+        <th>For Legacy Users (Py2)</th>
         <th>For Testers</th>
         <th>For Developers</th>
       </tr>
@@ -17,10 +18,13 @@
     <tbody>
       <tr>
         <td>
-          <a class="btn btn180 rounded green" href="https://mdipierro.pythonanywhere.com/examples/static/web2py_win.zip">For Windows</a>
+          <a class="btn btn180 rounded green" href="https://mdipierro.pythonanywhere.com/examples/static/web2py_win.zip">Windows binaries</a>
         </td>
         <td>
-          <a class="btn btn180 rounded yellow" href="https://mdipierro.pythonanywhere.com/examples/static/nightly/web2py_win.zip">For Windows</a>
+          <a class="btn btn180 rounded" href="https://mdipierro.pythonanywhere.com/examples/static/web2py_win_py2.zip">Windows binaries</a>
+        </td>
+        <td>
+          <a class="btn btn180 rounded yellow" href="https://mdipierro.pythonanywhere.com/examples/static/nightly/web2py_win.zip">Windows binaries</a>
         </td>
         <td>
           <a class="btn btn180 rounded red" href="http://github.com/web2py/web2py/">Git Repository</a>
@@ -28,17 +32,25 @@
       </tr>
       <tr>
         <td>
-          <a class="btn btn180 rounded green" href="https://mdipierro.pythonanywhere.com/examples/static/web2py_osx.zip">For Mac</a>
+          <a class="btn btn180 rounded green" href="https://mdipierro.pythonanywhere.com/examples/static/web2py_osx.zip">Mac binaries</a>
         </td>
         <td>
-          <a class="btn btn180 rounded yellow" href="https://mdipierro.pythonanywhere.com/examples/static/nightly/web2py_osx.zip">For Mac</a>
+          <a class="btn btn180 rounded" href="https://mdipierro.pythonanywhere.com/examples/static/web2py_osx_py2.zip">Mac binaries</a>
+        </td>
+        <td>
+          <a class="btn btn180 rounded yellow" href="https://mdipierro.pythonanywhere.com/examples/static/nightly/web2py_osx.zip">Mac binaries</a>
+        </td>
+        <td>
+          <a class="btn btn180 rounded" href="http://mdipierro.github.io/web2py/web2py_manual_5th.pdf">Manual</a>
         </td>
-        <td></td>
       </tr>
       <tr>
         <td>
           <a class="btn btn180 rounded green" href="https://mdipierro.pythonanywhere.com/examples/static/web2py_src.zip">Source Code</a>
         </td>
+        <td>
+          <a class="btn btn180 rounded" href="https://mdipierro.pythonanywhere.com/examples/static/web2py_src.zip">Source Code</a>
+        </td>
         <td>
           <a class="btn btn180 rounded yellow" href="https://mdipierro.pythonanywhere.com/examples/static/nightly/web2py_src.zip">Source Code</a>
         </td>
@@ -48,7 +60,8 @@
       </tr>
       <tr>
         <td>
-          <a class="btn btn180 rounded green" href="http://mdipierro.github.io/web2py/web2py_manual_5th.pdf">Manual</a>
+        </td>
+        <td>
         </td>
         <td>
           <a class="btn btn180 rounded" href="https://github.com/web2py/web2py/releases">Change Log</a>
@@ -62,13 +75,19 @@
 </center>
 
 <p style="text-align:left;">
-  The source code version works on Windows and most Unix systems, including <b>Linux</b>,  <b>BSD</b> and  <b>Mac</b> . It requires Python 2.6 (no more supported),  Python 2.7 (stable) or Python 3.5+ (recommended for new projects) already installed on your system.
-  There are also binary packages for Windows and Mac OS X. They include the Python 2.7 interpreter so you do not need to have it pre-installed.
+  The source code version works on Windows and most Unix systems, including <b>Linux</b>,  <b>BSD</b> and  <b>Mac</b> . It requires Python 3.5+ (recommended for new projects) 
+  or Python 2.7+ (stable, for use with legacy apps) already installed on your system.
+</p>
+<p style="text-align:left;">
+  There are also binary packages for Windows and MacOs. They include the Python interpreter version 3.7.4 or 2.7.16, so you do not need to have it pre-installed.
 </p>
 
 <h3>Instructions</h3>
-<p>With the binary packages, after download, just unzip it and then click on web2py.exe (windows) or web2py.app (osx).
-  If you prefer to run it from source with your own Python interpreter alreay installed, type:</p>
+<p>With the binary packages, after download, just unzip it and then click on web2py.exe (Windows) or web2py (MacOs).</p>
+<p>Note that on recent MacOs versions (10.12+) you could face problems in running the binary App program, due to the last changes to the security settings. 
+In this case, press the 'control' key + click on downloaded file and then 'open' it (confirm the warnings). Finally move the program in Applications and run it from there.
+</p>
+<p> If you prefer to run it from source with your own Python interpreter already installed, type:</p>
 {{=CODE("python web2py.py", language=None, counter='>', _class='boxCode')}}
 <p>or for more info type:</p>
 {{=CODE("python web2py.py -h", language=None, counter='>', _class='boxCode')}}

applications/welcome/static/js/web2py.js

@@ -283,7 +283,11 @@
             doc.ajaxSuccess(function (e, xhr) {
                 var redirect = xhr.getResponseHeader('web2py-redirect-location');
                 if (redirect !== null) {
-                    window.location = redirect;
+                    if (!redirect.endsWith('#')) {
+                        window.location.href = redirect;
+                    } else {
+                        window.location.reload();
+                    }
                 }
                 /* run this here only if this Ajax request is NOT for a web2py component. */
                 if (xhr.getResponseHeader('web2py-component-content') === null) {
@@ -335,7 +339,7 @@
                     } else {
                         formData = form.serialize(); // Fallback for older browsers.
                     }
-                    web2py.ajax_page('post', url, formData, target, form);
+                    web2py.ajax_page('post', url, formData, target);
 
                     e.preventDefault();
                 });
@@ -367,6 +371,24 @@
                     'data': data,
                     'processData': !isFormData,
                     'contentType': contentType,
+                    'xhr': function() {
+                        var xhr = new window.XMLHttpRequest();
+
+                        xhr.upload.addEventListener("progress", function(evt) {
+                            if (evt.lengthComputable) {
+                                var percentComplete = evt.loaded / evt.total;
+                                percentComplete = parseInt(percentComplete * 100);
+                                web2py.fire(element, 'w2p:uploadProgress', [percentComplete], target);
+
+                                if (percentComplete === 100) {
+                                    web2py.fire(element, 'w2p:uploadComplete', [], target);
+                                }
+
+                            }
+                        }, false);
+
+                        return xhr;
+                    },
                     'beforeSend': function (xhr, settings) {
                         xhr.setRequestHeader('web2py-component-location', document.location);
                         xhr.setRequestHeader('web2py-component-element', target);

extras/build_web2py/README_mac.md

@@ -0,0 +1,50 @@
+## MacOS binaries
+
+The MacOS binaries contain Python 3.7.3 (or 2.7.16) 64 bit with all the needed modules and the web2py in the specified version: 
+ you don't need anything else to run them on MacOS! After uncompressing the zip file, you just need to click on the web2py icon inside.  
+
+They were produced on MacOS Sierra 10.12.6 + security update 2019.001.
+
+## Full MacOS build recipe
+
+1. grab and install the official Python program: we've got version 3.7.3 or 2.7.16 (64 bit). If you've chosen python 2, change pip3
+with pip, and python3 with python in the following instructions...
+
+2. Open a terminal, update tools with:
+
+"python3 -m pip install --upgrade pip"  
+"pip3 install --upgrade setuptools"   
+
+
+3. install PyInstaller with:  
+sudo -H pip3 install pyinstaller (we've got PyInstaller-3.4 )
+
+4. additional (but not required) packages:  
+(only for python 2: install Homebrew from https://brew.sh/#install , then 'brew install unixodbc' )  
+pip3 install psycopg2-binary = psycopg2-2.7.7  
+pip3 install pyodbc = pyodbc-4.0.26-cp37-cp37m  
+pip3 install python-ldap (on the windows message, accept to install the "Command line developer tools"). Rerun:  
+pip3 install python-ldap  
+
+5. grab latest web2py source from https://mdipierro.pythonanywhere.com/examples/static/web2py_src.zip 
+ (you need at least 2.18.3 for needed changes in gluon\admin.py). Open it to uncompress, in this example on Desktop/web2py
+
+
+6. take the file build_web2py.py and web2py.mac.spec  from this folder and place it on the Desktop/web2py  folder  
+
+7. edit the file /Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/PyInstaller/hooks/hook-_tkinter.py 
+ and change one of its line according to https://github.com/pyinstaller/pyinstaller/pull/3830  
+
+8. (optional, for having a full working interactive shell) change the fake site.py module included within the PyInstaller installation 
+ with the content of the files web2py.site_37.py or web2py.site_27.py from this folder - see comments inside these files for details
+
+9. open a terminal, goto Desktop/web2py and run:  
+  
+python3 build_web2py.py
+
+10. if everything is fine, you'll obtain web2py_macos.zip on the Desktop/web2py  folder. Inside it, there is the web2py program with 
+ both the CMD version and the APP version.
+
+## Gothca
+
+Unfortunately, the APP version is still not working - see https://github.com/pyinstaller/pyinstaller/issues/3820 . 

extras/build_web2py/README_win.md

@@ -0,0 +1,47 @@
+## Windows binaries
+
+The windows binaries contain Python 64 bit version 3.7.3 or 2.7.16 with all the needed modules and the web2py in the specified version.
+You don't need anything else to run them on Windows.  
+At least on Windows 7, if you get an error stating that "api-ms-win-crt-runtime-l1-1-0.dll is missing" you have only to install the
+free and official "Visual C++ Redistributable for Visual Studio" as described later
+
+
+## Full Windows build recipe
+
+1. get a clean Windows 10 (Windows 10 Professional English build 1809 64 bit, under Virtualbox in our case)
+2. grab and install the official Python program: we've got version 3.7.3 or 2.7.16, 64 bit 
+(https://www.python.org/ftp/python/3.7.2/python-3.7.2-amd64.exe ) + select  "add Python 3.7 to PATH" during its setup if Python 3. 
+For Python 2 you need to manually add the folders for python27 and python27\Scripts to the system path.
+3. update tools with   
+"python -m pip install --upgrade pip"  
+"pip install --upgrade setuptools"  
+4. download and install python-win32, which is needed for web2py to work with all features enabled 
+(https://github.com/mhammond/pywin32/releases/download/b224/pywin32-224.win-amd64-py3.7.exe)
+5. grab latest web2py source from https://mdipierro.pythonanywhere.com/examples/static/web2py_src.zip (you need at least 2.18.3 for 
+needed changes in gluon\admin.py). Unzip it in a dedicated folder, in this example C:\web2py - so that you have 
+C:\web2py\web2py.py inside)
+6. install PyInstaller with:  
+        pip install pyinstaller  (we've got PyInstaller-3.4.tar.gz )  
+7. download and install the free Microsoft Visual C++ Redistributable per Visual Studio 2017, 64 bit version, from 
+ https://aka.ms/vs/15/release/vc_redist.x64.exe  
+8. additional (but not required) packages to work better in the Windows world:  
+pip install psycopg2 = psycopg2-2.7.7-cp37-cp37m-win_amd64.whl  
+pip install pyodbc = pyodbc-4.0.26-cp37-cp37m-win_amd64.whl  
+download the file python_ldap-3.1.0-cp37-cp37m-win_amd64.whl from https://www.lfd.uci.edu/~gohlke/pythonlibs/ and install it from that 
+folder with the command 'pip install python_ldap-3.1.0-cp37-cp37m-win_amd64.whl'  
+
+9. copy build_web2py.py, web2py.win.spec and web2py.win_no_console.spec from this folder to C:\web2py\  
+10. (only for python 2) - due to a PyInstaller bug, you need to manually change the file gluon\rocket.py, line 26, from IS_JYTHON 
+= platform.system() == 'Java'  to  IS_JYTHON = False
+11. (optional, for having a full working interactive shell) change the fake site.py module included within the PyInstaller installation
+with the content of the files web2py.site_37.py or web2py.site_27.py from this folder - see comments inside these files for details 
+12. open a CMD and go to C:\web2py. Run:
+
+    python build_web2py.py
+
+If everything goes fine, you'll obtain the 64 bit binary build zipped as C:\web2py\web2py_win.zip.
+If you try to run it in a 32 bit Windows system, you'll correctly get a 'web2py.exe not a valid Win32 application' error message.
+
+## Gothca:
+- at least on Windows 7, you can get an error stating that "api-ms-win-crt-runtime-l1-1-0.dll is missing". You can easily resolve it by
+installing "Visual C++ Redistributable for Visual Studio" described earlier

gluon/authapi.py

@@ -803,7 +803,7 @@ class AuthAPI(object):
 
         # Finally verify the password
         passfield = settings.password_field
-        password = table_user[passfield].validate(kwargs.get(passfield, ''))[0]
+        password = table_user[passfield].validate(kwargs.get(passfield, ''), None)[0]
 
         if password == user[passfield]:
             self.login_user(user)

gluon/compileapp.py

@@ -565,7 +565,7 @@ def run_models_in(environment):
 
 TEST_CODE = r"""
 def _TEST():
-    import doctest, sys, cStringIO, types, cgi, gluon.fileutils
+    import doctest, sys, cStringIO, types, gluon.fileutils
     if not gluon.fileutils.check_credentials(request):
         raise HTTP(401, web2py_error='invalid credentials')
     stdout = sys.stdout

gluon/console.py

@@ -266,6 +266,12 @@ web2py will attempt to run a GUI to ask for it when starting the web server
         '(default is %(default)s), see -S above. NOTE: when the APP_ENV '
         'argument of -S include a controller c automatic import of '
         'models is always enabled')
+    g.add_argument('--fake_migrate',
+                   default=False,
+                   action='store_true',
+                   help=
+                   'force DAL to fake migrate all tables; '
+                   'monkeypatch in the DAL class to force _fake_migrate=True')
     g.add_argument('--force_migrate', '--force-migrate',
                    default=False,
                    action='store_true', help=

gluon/contrib/login_methods/freeipa_auth.py

@@ -0,0 +1,59 @@
+import sys
+import logging
+
+try:
+    import ldap
+
+    ldap.set_option(ldap.OPT_X_TLS_REQUIRE_CERT, ldap.OPT_X_TLS_NEVER)
+except Exception as detail:
+    logging.error('missing ldap, try "pip install python-ldap"')
+    raise detail
+
+
+def freeipa_auth(server, basedn, group):
+    """
+    custom module for freeIPA auth in web2py
+    server: freeipa ip
+    base_dn: root of ldap tree containing user & groups
+    group: group authing user has to be a member of
+
+   """
+    logger = logging.getLogger("web2py.auth.freeipa_auth")
+
+    def freeipa_auth_aux(username, password):
+        if password == "" or username == "":
+            logger.warning("blank username / password not allowed")
+            return False
+
+        bind_user_base = "uid=" + username + ",cn=users," + basedn
+        ldap_filter = "memberof=cn=" + group + ",cn=groups," + basedn
+
+        session = ldap.initialize("ldaps://" + server + ":636")
+        try:
+            session.bind_s(bind_user_base, password)
+        except ldap.LDAPError:
+            import traceback
+
+            logger.warning("[%s] Error in ldap bind" % str(username))
+            logger.debug(traceback.format_exc())
+            return False
+
+        try:
+            result = session.search_s(
+                bind_user_base, ldap.SCOPE_SUBTREE, ldap_filter, ["member"]
+            )
+            session.unbind()
+        except ldap.LDAPError as detail:
+            logger.warning(
+                "ldap_auth: searc %s for %s resulted in %s: %s\n"
+                % (bind_user_base, ldap_filter, exc_type, exc_value)
+            )
+
+        try:
+            if result == list():
+                return False
+            return True
+        except:
+            return False
+
+    return freeipa_auth_aux

gluon/contrib/login_methods/saml2_auth.py

@@ -150,14 +150,14 @@ class Saml2Auth(object):
         self.config_file = config_file
         self.maps = maps
 
-    # URL for redirecting users to when they sign out
+        # URL for redirecting users to when they sign out
         self.saml_logout_url = logout_url
 
         # URL to let users change their password in the IDP system
         self.saml_change_password_url = change_password_url
     
-    # URL to specify an IDP if using federation metadata or an MDQ
-    self.entityid = entityid
+        # URL to specify an IDP if using federation metadata or an MDQ
+        self.entityid = entityid
 
     def login_url(self, next="/"):
         d = saml2_handler(current.session, current.request, entityid=self.entityid)

gluon/contrib/markmin/markmin2latex.py

@@ -237,6 +237,7 @@ WRAPPER = """
 \\usepackage{graphicx}
 \\usepackage{grffile}
 \\usepackage[utf8x]{inputenc}
+\\usepackage{textgreek}
 \\definecolor{lg}{rgb}{0.9,0.9,0.9}
 \\definecolor{dg}{rgb}{0.3,0.3,0.3}
 \\def\\ft{\\small\\tt}

gluon/contrib/markmin/markmin2pdf.py

@@ -80,7 +80,7 @@ def latex2pdf(latex, pdflatex='pdflatex', passes=3):
             outfile.close()
         re_errors = re.compile('^\!(.*)$', re.M)
         re_warnings = re.compile('^LaTeX Warning\:(.*)$', re.M)
-        flog = open(logname)
+        flog = open(logname,encoding="utf-8")
         try:
             loglines = flog.read()
         finally:

gluon/contrib/redis_cache.py

@@ -208,12 +208,12 @@ class RedisClient(object):
 
     def retry_call(self, key, f, time_expire, with_lock):
         self.RETRIES += 1
-        if self.RETRIES <= self.MAX_RETRIES:
-            logger.error("sleeping %s seconds before reconnecting" % (2 * self.RETRIES))
-            time.sleep(2 * self.RETRIES)
+        if self.RETRIES <= self.MAX_RETRIES:            
             if self.fail_gracefully:
                 self.RETRIES = 0
                 return f()
+            logger.error("sleeping %s seconds before reconnecting" % (2 * self.RETRIES))            
+            time.sleep(2 * self.RETRIES)
             return self.__call__(key, f, time_expire, with_lock)
         else:
             self.RETRIES = 0

gluon/contrib/redis_session.py

@@ -13,7 +13,7 @@ from gluon import current
 from gluon.storage import Storage
 from gluon.contrib.redis_utils import acquire_lock, release_lock
 from gluon.contrib.redis_utils import register_release_lock
-from gluon._compat import to_bytes
+from gluon._compat import to_native
 
 logger = logging.getLogger("web2py.session.redis")
 
@@ -182,11 +182,11 @@ class MockQuery(object):
             key = self.keyprefix + ':' + str(self.value)
             if self.with_lock:
                 acquire_lock(self.db.r_server, key + ':lock', self.value, 2)
-            rtn = self.db.r_server.hgetall(key)
+            rtn = {to_native(k.decode): v for k, v in self.db.r_server.hgetall(key).items()}
             if rtn:
                 if self.unique_key:
                     # make sure the id and unique_key are correct
-                    if rtn['unique_key'] == self.unique_key:
+                    if rtn['unique_key'] == to_native(self.unique_key):
                         rtn['update_record'] = self.update  # update record support
                     else:
                         rtn = None

gluon/contrib/simplejsonrpc.py

@@ -30,7 +30,7 @@ else:
     from io import StringIO
 import random
 import json
-
+from gluon._compat import basestring
 
 class JSONRPCError(RuntimeError):
     "Error object for remote procedure call fail"

gluon/globals.py

@@ -13,7 +13,7 @@ Contains the classes for the global used variables:
 
 """
 from gluon._compat import pickle, StringIO, copyreg, Cookie, urlparse, PY2, iteritems, to_unicode, to_native, \
-    to_bytes, unicodeT, long, hashlib_md5, urllib_quote
+    to_bytes, unicodeT, long, hashlib_md5, urllib_quote, to_native
 from gluon.storage import Storage, List
 from gluon.streamer import streamer, stream_file_or_304_or_206, DEFAULT_CHUNK_SIZE
 from gluon.contenttype import contenttype
@@ -1055,7 +1055,7 @@ class Session(Storage):
             if record_id.isdigit() and long(record_id) > 0:
                 new_unique_key = web2py_uuid()
                 row = table(record_id)
-                if row and row['unique_key'] == unique_key:
+                if row and to_native(row['unique_key']) == to_native(unique_key):
                     table._db(table.id == record_id).update(unique_key=new_unique_key)
                 else:
                     record_id = None
@@ -1231,9 +1231,9 @@ class Session(Storage):
 
         session_pickled = response.session_pickled or pickle.dumps(self, pickle.HIGHEST_PROTOCOL)
 
-        dd = dict(locked=False,
+        dd = dict(locked=0,
                   client_ip=response.session_client,
-                  modified_datetime=request.now,
+                  modified_datetime=request.now.isoformat(),
                   session_data=session_pickled,
                   unique_key=unique_key)
         if record_id:

gluon/languages.py

@@ -16,7 +16,6 @@ import re
 import sys
 import pkgutil
 import logging
-from cgi import escape
 from threading import RLock
 
 from pydal._compat import copyreg, PY2, maketrans, iterkeys, unicodeT, to_unicode, to_bytes, iteritems, to_native, pjoin

gluon/scheduler.py

@@ -530,7 +530,7 @@ class IS_CRONLINE(object):
     def __init__(self, error_message=None):
         self.error_message = error_message
 
-    def __call__(self, value):
+    def __call__(self, value, record_id=None):
         recur = CronParser(value, datetime.datetime.now())
         try:
             recur.next()
@@ -550,7 +550,7 @@ class TYPE(object):
         self.myclass = myclass
         self.parse = parse
 
-    def __call__(self, value):
+    def __call__(self, value, record_id=None):
         from gluon import current
         try:
             obj = loads(value)
@@ -596,12 +596,12 @@ class Scheduler(threading.Thread):
         utc_time(bool): do all datetime calculations assuming UTC as the
             timezone. Remember to pass `start_time` and `stop_time` to tasks
             accordingly
-
+        use_spawn(bool): use spawn for subprocess (only useable with python3)
     """
 
     def __init__(self, db, tasks=None, migrate=True,
                  worker_name=None, group_names=None, heartbeat=HEARTBEAT,
-                 max_empty_runs=0, discard_results=False, utc_time=False):
+                 max_empty_runs=0, discard_results=False, utc_time=False, use_spawn=False):
 
         threading.Thread.__init__(self)
         self.setDaemon(True)
@@ -639,6 +639,7 @@ class Scheduler(threading.Thread):
         current._scheduler = self
 
         self.define_tables(db, migrate=migrate)
+        self.use_spawn = use_spawn
 
     def execute(self, task):
         """Start the background process.
@@ -649,10 +650,19 @@ class Scheduler(threading.Thread):
         Returns:
             a `TaskReport` object
         """
-        outq = multiprocessing.Queue()
-        retq = multiprocessing.Queue(maxsize=1)
-        self.process = p = \
-            multiprocessing.Process(target=executor, args=(retq, task, outq))
+        outq = None
+        retq = None
+        if (self.use_spawn and not PY2):
+            ctx = multiprocessing.get_context('spawn')
+            outq = ctx.Queue()
+            retq = ctx.Queue(maxsize=1)
+            sel.process = p = ctx.Process(target=executor, args=(retq, task, outq))
+        else:
+            outq = multiprocessing.Queue()
+            retq = multiprocessing.Queue(maxsize=1)
+            self.process = p = \
+                multiprocessing.Process(target=executor, args=(retq, task, outq))
+
         self.process_queues = (retq, outq)
 
         logger.debug('   task starting')

gluon/serializers.py

@@ -82,7 +82,7 @@ def custom_json(o):
     elif isinstance(o, integer_types):
         return int(o)
     elif isinstance(o, decimal.Decimal):
-        return str(o)
+        return float(o)
     elif isinstance(o, (bytes, bytearray)):
         return str(o)
     elif isinstance(o, lazyT):

gluon/shell.py

@@ -217,7 +217,8 @@ def run(
     python_code=None,
     cron_job=False,
     scheduler_job=False,
-    force_migrate=False):
+    force_migrate=False,
+    fake_migrate=False):
     """
     Start interactive shell or run Python script (startfile) in web2py
     controller environment. appname is formatted like:
@@ -233,7 +234,7 @@ def run(
     errmsg = 'invalid application name: %s' % appname
     if not a:
         die(errmsg, error_preamble=False)
-    adir = os.path.join('applications', a)
+    adir = os.path.absdir(os.path.join('applications', a))
 
     if not os.path.exists(adir):
         if not cron_job and not scheduler_job and \
@@ -248,6 +249,7 @@ def run(
             fileutils.create_app(adir)
 
     if force_migrate:
+        c = 'appadmin' # Load all models (hack already used for appadmin controller)
         import_models = True
         from gluon.dal import DAL
         orig_init = DAL.__init__
@@ -255,6 +257,7 @@ def run(
         def custom_init(*args, **kwargs):
             kwargs['migrate_enabled'] = True
             kwargs['migrate'] = True
+            kwargs['fake_migrate'] = fake_migrate
             logger.info('Forcing migrate_enabled=True')
             orig_init(*args, **kwargs)
 
@@ -270,7 +273,7 @@ def run(
     if vars:
         # underscore necessary because request.vars is a property
         extra_request['_vars'] = vars
-    _env = env(a, c=c, f=f, import_models=import_models, extra_request=extra_request)
+    _env = env(a, c=c, f=f, import_models=import_models, extra_request=extra_request, dir=adir)
 
     if c:
         pyfile = os.path.join('applications', a, 'controllers', c + '.py')

gluon/tests/test_serializers.py

@@ -44,7 +44,7 @@ class TestSerializers(unittest.TestCase):
         # self.assertEqual(json(1), json(1))
         # decimal stringified
         obj = {'a': decimal.Decimal('4.312312312312')}
-        self.assertEqual(json(obj), u'{"a": "4.312312312312"}')
+        self.assertEqual(json(obj), u'{"a": 4.312312312312}')
         # lazyT translated
         T = TranslatorFactory('', 'en')
         lazy_translation = T('abc')

gluon/tools.py

@@ -2301,7 +2301,7 @@ class Auth(AuthAPI):
         # in this case they will have to reset their password to login
         if fields.get(settings.passfield):
             fields[settings.passfield] = \
-                settings.table_user[settings.passfield].validate(fields[settings.passfield])[0]
+                settings.table_user[settings.passfield].validate(fields[settings.passfield], None)[0]
         if not fields.get(settings.userfield):
             raise ValueError('register_bare: userfield not provided or invalid')
         user = self.get_or_create_user(fields, login=False, get=False,
@@ -2638,9 +2638,7 @@ class Auth(AuthAPI):
                         # invalid login
                         session.flash = specific_error if self.settings.login_specify_error else self.messages.invalid_login
                         callback(onfail, None)
-                        if 'password' in request.post_vars:
-                            del request.post_vars['password']
-                        redirect(self.url(args=request.args, vars=request.vars),client_side=settings.client_side)
+                        redirect(self.url(args=request.args, vars=request.get_vars),client_side=settings.client_side)
 
             else:  # use a central authentication server
                 cas = settings.login_form
@@ -3173,12 +3171,12 @@ class Auth(AuthAPI):
                         formname='retrieve_password', dbio=False,
                         onvalidation=onvalidation, hideerror=self.settings.hideerror):
             user = table_user(email=form.vars.email)
-            key = user.registration_key
             if not user:
                 current.session.flash = \
                     self.messages.invalid_email
                 redirect(self.url(args=request.args))
-            elif key in ('pending', 'disabled', 'blocked') or (key or '').startswith('pending'):
+            key = user.registration_key
+            if key in ('pending', 'disabled', 'blocked') or (key or '').startswith('pending'):
                 current.session.flash = \
                     self.messages.registration_pending
                 redirect(self.url(args=request.args))

gluon/widget.py

@@ -741,7 +741,8 @@ def start():
         sys.argv = [options.run or ''] + options.args
         run(options.shell, plain=options.plain, bpython=options.bpython,
             import_models=options.import_models, startfile=options.run,
-            cron_job=options.cron_job, force_migrate=options.force_migrate)
+            cron_job=options.cron_job, force_migrate=options.force_migrate,
+            fake_migrate=options.fake_migrate)
         return
 
     # set size of cron thread pools

scripts/migrator.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 '''
 To use, e.g. python .\web2py.py -S APPNAME --force_migrate
+To use, e.g. python .\web2py.py -S APPNAME --force_migrate --fake_migrate
 '''
 
 import logging
@@ -31,5 +32,6 @@ for db_name in databases:
     for table_name in tables:
         # Force migration of lazy tables
         logger.debug("Ensuring migration of table '%s'", table_name)
-        db(db[table_name]).isempty()
+        table = db[table_name]
+        db(table).isempty()
     db.commit()