fixed test because of layout change

fixed toolbar/configuration issue
fixed layout in lack of configuration
2017-11-14 00:55:59 -06:00 · 2017-11-14 00:50:44 -06:00 · 2017-11-14 00:47:40 -06:00 · 2017-11-13 23:50:20 -06:00 · 2017-11-13 23:48:54 -06:00 · 2017-11-13 23:29:34 -06:00
78 changed files with 1423 additions and 824 deletions
@@ -4,11 +4,18 @@ sudo: required

 cache: pip

+dist: "trusty"
+
 python:
  - '2.7'
-  - 'pypy'
  - '3.5'
  - '3.6'
+  - 'pypy-5.3.1'
+  - 'pypy3.5-5.7.1-beta'
+
+matrix:
+  allow_failures:
+    - python: 'pypy3.5-5.7.1-beta'

 install:
  - pip install -e .
@@ -33,3 +40,6 @@ notifications:

 addons:
  postgresql: "9.4"
+  apt:
+    packages:
+      - postgresql-9.4-postgis-2.3
@@ -1,7 +1,18 @@
-## 2.16.0b1
+## 2.16.1
+- pydal 17.11
+- bootstrap 4 
+- better welcome examples
+- many bug fixes
+
+## 2.15.1-4
+- pydal 17.08
+- dropped support for python 2.6
+- dropped web shell
 - experimental python 3 support
 - experimental authapi for service login
+- allow ajax file uploads
 - more tests
+- more pep8 compliance
 - d3.js model visulization
 - improved scheduler
 - is_email support for internationalized Domain Names
@@ -20,6 +31,13 @@
 - Updated fpdf to latest version
 - JWT support
 - import fabfile for remote deployment
+- scheduler new feature: you can now specify intervals with cron
+- gluon/* removed from sys.path. Applications relying on statements like e.g.
+  "from storage import Storage"
+  will need to be rewritten with
+  "from gluon.storage import Storage"
+- tests can only be run with the usual web2py.py --run_system_tests OR with
+  python -m unittest -v gluon.tests on the root dir
 - jQuery 3.2.1
 - PyDAL 17.07 including:
  allow jsonb support for postgres
@@ -30,21 +48,8 @@
  improved Teradata support
  improved mongodb support
  overall refactoring
-  experimental support for Google Cloud SQL v2 (TODO)
-
-## 2.15.x
- web2py does not support python 2.6 anymore
- py3.5 syntax compatible (see #1353 for details)
- dropped web shell from admin
- scheduler new feature: you can now specify intervals with cron
- gluon/* removed from sys.path. Applications relying on statements like e.g.
-  "from storage import Storage"
-  will need to be rewritten with
-  "from gluon.storage import Storage"
- tests can only be run with the usual web2py.py --run_system_tests OR with
-  python -m unittest -v gluon.tests on the root dir
- updated pymysql driver
-
+  experimental support for Google Cloud SQL v2
+  new pymysql driver

 ## 2.14.6

@@ -30,11 +30,7 @@ update:
 	wget -O gluon/contrib/feedparser.py http://feedparser.googlecode.com/svn/trunk/feedparser/feedparser.py
 	wget -O gluon/contrib/simplejsonrpc.py http://rad2py.googlecode.com/hg/ide2py/simplejsonrpc.py
 	echo "remember that pymysql was tweaked"
-src:
-	### Use semantic versioning
-	echo 'Version 2.14.6-stable+timestamp.'`date +%Y.%m.%d.%H.%M.%S` > VERSION
-	### rm -f all junk files
-	make clean
+rmfiles:
 	### clean up baisc apps
 	rm -f routes.py 
 	rm -rf applications/*/sessions/*
@@ -46,6 +42,12 @@ src:
 	rm -rf applications/admin/uploads/*                 
 	rm -rf applications/welcome/uploads/*               
 	rm -rf applications/examples/uploads/* 
+src:
+	### Use semantic versioning
+	echo 'Version 2.16.1-stable+timestamp.'`date +%Y.%m.%d.%H.%M.%S` > VERSION
+	### rm -f all junk files
+	#make clean
+	# make rmfiles
 	### make welcome layout and appadmin the default
 	cp applications/welcome/views/appadmin.html applications/admin/views
 	cp applications/welcome/views/appadmin.html applications/examples/views
@@ -54,7 +56,7 @@ src:
 	### build web2py_src.zip
 	echo '' > NEWINSTALL
 	mv web2py_src.zip web2py_src_old.zip | echo 'no old'
-	cd ..; zip -r web2py/web2py_src.zip web2py/web2py.py web2py/anyserver.py web2py/gluon/* web2py/extras/* web2py/handlers/* web2py/examples/* web2py/README.markdown  web2py/LICENSE web2py/CHANGELOG web2py/NEWINSTALL web2py/VERSION web2py/MANIFEST.in web2py/scripts/*.sh web2py/scripts/*.py web2py/applications/admin web2py/applications/examples/ web2py/applications/welcome web2py/applications/__init__.py web2py/site-packages/__init__.py web2py/gluon/tests/*.sh web2py/gluon/tests/*.py
+	cd ..; zip -r --exclude=**.git** web2py/web2py_src.zip web2py/web2py.py web2py/anyserver.py web2py/fabfile.py web2py/gluon/* web2py/extras/* web2py/handlers/* web2py/examples/* web2py/README.markdown  web2py/LICENSE web2py/CHANGELOG web2py/NEWINSTALL web2py/VERSION web2py/MANIFEST.in web2py/scripts/*.sh web2py/scripts/*.py web2py/applications/admin web2py/applications/examples/ web2py/applications/welcome web2py/applications/__init__.py web2py/site-packages/__init__.py web2py/gluon/tests/*.sh web2py/gluon/tests/*.py

 mdp:
 	make src
@@ -97,6 +99,8 @@ win:
 	cp -r applications/welcome ../web2py_win/web2py/applications
 	cp -r applications/examples ../web2py_win/web2py/applications
 	cp applications/__init__.py ../web2py_win/web2py/applications
+	# per https://github.com/web2py/web2py/issues/1716
+	mv ../web2py_win/web2py/_ssl.pyd ../web2py_win/web2py/_ssl.pyd.legacy | echo 'done'
 	cd ../web2py_win; zip -r web2py_win.zip web2py
 	mv ../web2py_win/web2py_win.zip .
 run:
@@ -1 +1 @@
-Version 2.14.6-stable+timestamp.2016.05.09.19.18.48
+Version 2.16.1-stable+timestamp.2017.11.13.23.50.07
@@ -657,39 +657,38 @@ def d3_graph_model():
    Create a list of table dicts, called "nodes"
    """
    
-    data = {}
    nodes = []
    links = []

-    subgraphs = dict()
+    for database in databases:
+        db = eval_in_global_env(database)
+        for tablename in db.tables:
+            fields = []
+            for field in db[tablename]:
+                f_type = field.type
+                if not isinstance(f_type,str):
+                    disp = ' '
+                elif f_type == 'string':
+                    disp =  field.length
+                elif f_type == 'id':
+                    disp =  "PK"
+                elif f_type.startswith('reference') or \
+                    f_type.startswith('list:reference'):
+                    disp = "FK"
+                else:
+                    disp = ' '
+                fields.append(dict(name= field.name, type=field.type, disp = disp))

-    for tablename in db.tables:
-        fields = []
-        for field in db[tablename]:
-            f_type = field.type
-            if not isinstance(f_type,str):
-                disp = ' '
-            elif f_type == 'string':
-                disp =  field.length
-            elif f_type == 'id':
-                disp =  "PK"
-            elif f_type.startswith('reference') or \
-                f_type.startswith('list:reference'):
-                disp = "FK"
-            else:
-                disp = ' '
-            fields.append(dict(name= field.name, type=field.type, disp = disp))
+                if isinstance(f_type,str) and (
+                    f_type.startswith('reference') or
+                    f_type.startswith('list:reference')):
+                    referenced_table = f_type.split()[1].split('.')[0]

-            if isinstance(f_type,str) and (
-                f_type.startswith('reference') or
-                f_type.startswith('list:reference')):
-                referenced_table = f_type.split()[1].split('.')[0]
+                    links.append(dict(source=tablename, target = referenced_table))

-                links.append(dict(source=tablename, target = referenced_table))
-
-        nodes.append(dict(name=tablename, type="table", fields = fields))
+            nodes.append(dict(name=tablename, type="table", fields = fields))

    # d3 v4 allows individual modules to be specified.  The complete d3 library is included below.
-    response.files.append(URL('static','js/d3.min.js'))
-    response.files.append(URL('static','js/d3_graph.js'))
-    return dict(databases=databases, nodes=nodes, links=links)
+    response.files.append(URL('admin','static','js/d3.min.js'))
+    response.files.append(URL('admin','static','js/d3_graph.js'))
+    return dict(databases=databases, nodes=nodes, links=links)
@@ -12,6 +12,10 @@
        $.error('web2py.js has already been loaded!');
    }

+    var FORMDATA_IS_SUPPORTED = typeof(FormData) !== 'undefined';
+    var animateIn = 'fadeIn';
+    // var animateIn = 'slideDown';
+
    String.prototype.reverse = function () {
        return this.split('').reverse().join('');
    };
@@ -178,7 +182,7 @@
        },
        /* manage errors in forms */
        manage_errors: function (target) {
-            $('div.error', target).hide().slideDown('slow');
+            $('div.error', target).hide()[animateIn]('slow');
        },
        after_ajax: function (xhr) {
            /* called whenever an ajax request completes */
@@ -263,13 +267,17 @@
                }
            });
            /* help preventing double form submission for normal form (not LOADed) */
-            $(doc).on('submit', 'form', function () {
-                var submit_button = $(this).find(web2py.formInputClickSelector);
-                web2py.disableElement(submit_button);
+            $(doc).on('submit', 'form', function (e) {
+                var submit_buttons = $(this).find(web2py.formInputClickSelector);
+                submit_buttons.each(function() {
+                    web2py.disableElement($(this));
+                })
                /* safeguard in case the form doesn't trigger a refresh,
                see https://github.com/web2py/web2py/issues/1100 */
                setTimeout(function () {
-                    web2py.enableElement(submit_button);
+                    submit_buttons.each(function() {
+                        web2py.enableElement($(this));
+                    });
                }, 5000);
            });
            doc.ajaxSuccess(function (e, xhr) {
@@ -320,7 +328,15 @@
                form.submit(function (e) {
                    web2py.disableElement(form.find(web2py.formInputClickSelector));
                    web2py.hide_flash();
-                    web2py.ajax_page('post', url, form.serialize(), target, form);
+
+                    var formData;
+                    if (FORMDATA_IS_SUPPORTED) {
+                        formData = new FormData(form[0]); // Allows file uploads.
+                    } else {
+                        formData = form.serialize(); // Fallback for older browsers.
+                    }
+                    web2py.ajax_page('post', url, formData, target, form);
+
                    e.preventDefault();
                });
                form.on('click', web2py.formInputClickSelector, function (e) {
@@ -339,11 +355,18 @@
            if (web2py.isUndefined(element)) element = $(document);
            /* if target is not there, fill it with something that there isn't in the page*/
            if (web2py.isUndefined(target) || target === '') target = 'w2p_none';
+
+            /* processData and contentType must be set to false when passing a FormData
+               object to jQuery.ajax. */
+            var isFormData = Object.prototype.toString.call(data) === '[object FormData]';
+            var contentType = isFormData ? false : 'application/x-www-form-urlencoded; charset=UTF-8';
            if (web2py.fire(element, 'ajax:before', null, target)) { /*test a usecase, should stop here if returns false */
                $.ajax({
                    'type': method,
                    'url': action,
                    'data': data,
+                    'processData': !isFormData,
+                    'contentType': contentType,
                    'beforeSend': function (xhr, settings) {
                        xhr.setRequestHeader('web2py-component-location', document.location);
                        xhr.setRequestHeader('web2py-component-element', target);
@@ -595,7 +618,7 @@
            var flash = $('.w2p_flash');
            web2py.hide_flash();
            flash.html(message).addClass(status);
-            if (flash.html()) flash.append('<span id="closeflash"> &times; </span>').slideDown();
+            if (flash.html()) flash.append('<span id="closeflash"> &times; </span>')[animateIn]();
        },
        hide_flash: function () {
            $('.w2p_flash').fadeOut(0).html('');
@@ -609,7 +632,7 @@
                for (var k = 0; k < triggers[id].length; k++) {
                    var dep = $('#' + triggers[id][k], target);
                    var tr = $('#' + triggers[id][k] + '__row', target);
-                    if (t.is(dep.attr('data-show-if'))) tr.slideDown();
+                    if (t.is(dep.attr('data-show-if'))) tr[animateIn]();
                    else tr.hide();
                }
            };
@@ -699,8 +722,9 @@
            });
        },
        /* Disables form elements:
+        - Does not disable elements with 'data-w2p_disable' attribute
        - Caches element value in 'w2p_enable_with' data store
-        - Replaces element text with value of 'data-disable-with' attribute
+        - Replaces element text with value of 'data-w2p_disable_with' attribute
        - Sets disabled property to true
        */
        disableFormElements: function (form) {
@@ -712,13 +736,15 @@
                if (!web2py.isUndefined(disable)) {
                    return false;
                }
-                if (web2py.isUndefined(disable_with)) {
-                    element.data('w2p_disable_with', element[method]());
+                if (!element.is(':file')) { // Altering file input values is not allowed.
+                    if (web2py.isUndefined(disable_with)) {
+                        element.data('w2p_disable_with', element[method]());
+                    }
+                    if (web2py.isUndefined(element.data('w2p_enable_with'))) {
+                        element.data('w2p_enable_with', element[method]());
+                    }
+                    element[method](element.data('w2p_disable_with'));
                }
-                if (web2py.isUndefined(element.data('w2p_enable_with'))) {
-                    element.data('w2p_enable_with', element[method]());
-                }
-                element[method](element.data('w2p_disable_with'));
                element.prop('disabled', true);
            });
        },
@@ -799,4 +825,4 @@ web2py_event_handlers = jQuery.web2py.event_handlers;
 web2py_trap_link = jQuery.web2py.trap_link;
 web2py_calc_entropy = jQuery.web2py.calc_entropy;
 */
-/* compatibility code - end*/
+/* compatibility code - end*/
@@ -239,7 +239,7 @@
    {{=T("No databases in this application")}}
  {{else:}}    
    <div id="vis"></div>
-      <link rel="stylesheet" href="{{=URL('static','css/d3_graph.css')}}"/>
+      <link rel="stylesheet" href="{{=URL('admin','static','css/d3_graph.css')}}"/>
      <script>
        // Define the d3 input data
        {{from gluon.serializers import json }}
@@ -46,7 +46,7 @@
      </td>
      <td>
        <div class="btn-group">
-          <a class="btn dropdown-toggle" data-toggle="dropdown" href="#">
+          <a class="btn dropdown-toggle" data-toggle="dropdown">
        {{=T('Manage')}}
        <span class="caret"></span>
          </a>
@@ -657,37 +657,36 @@ def d3_graph_model():
    Create a list of table dicts, called "nodes"
    """
    
-    data = {}
    nodes = []
    links = []

-    subgraphs = dict()
+    for database in databases:
+        db = eval_in_global_env(database)
+        for tablename in db.tables:
+            fields = []
+            for field in db[tablename]:
+                f_type = field.type
+                if not isinstance(f_type,str):
+                    disp = ' '
+                elif f_type == 'string':
+                    disp =  field.length
+                elif f_type == 'id':
+                    disp =  "PK"
+                elif f_type.startswith('reference') or \
+                    f_type.startswith('list:reference'):
+                    disp = "FK"
+                else:
+                    disp = ' '
+                fields.append(dict(name= field.name, type=field.type, disp = disp))

-    for tablename in db.tables:
-        fields = []
-        for field in db[tablename]:
-            f_type = field.type
-            if not isinstance(f_type,str):
-                disp = ' '
-            elif f_type == 'string':
-                disp =  field.length
-            elif f_type == 'id':
-                disp =  "PK"
-            elif f_type.startswith('reference') or \
-                f_type.startswith('list:reference'):
-                disp = "FK"
-            else:
-                disp = ' '
-            fields.append(dict(name= field.name, type=field.type, disp = disp))
+                if isinstance(f_type,str) and (
+                    f_type.startswith('reference') or
+                    f_type.startswith('list:reference')):
+                    referenced_table = f_type.split()[1].split('.')[0]

-            if isinstance(f_type,str) and (
-                f_type.startswith('reference') or
-                f_type.startswith('list:reference')):
-                referenced_table = f_type.split()[1].split('.')[0]
+                    links.append(dict(source=tablename, target = referenced_table))

-                links.append(dict(source=tablename, target = referenced_table))
-
-        nodes.append(dict(name=tablename, type="table", fields = fields))
+            nodes.append(dict(name=tablename, type="table", fields = fields))

    # d3 v4 allows individual modules to be specified.  The complete d3 library is included below.
    response.files.append(URL('admin','static','js/d3.min.js'))
@@ -3,9 +3,9 @@
 - **Created by a community of professionals** and University professors in Computer Science and Software Engineering.
 - **Always backward compatible.** We have not broken backward compatibility since version 1.0 in 2007, and we pledge not to break it in the future.
 - **Easy to run.** It requires no installation and no configuration.
- **Runs on** Windows, Mac, Unix/Linux, Google App Engine, Amazon EC2, and almost any web hosting via Python 2.5/2.6/2.7/pypy, or Java with Jython.
- **Runs with** Apache, Lighttpd, Cherokee and almost any other web server via CGI, FastCGI, WSGI, mod_proxy, and/or mod_python. It can embed third party WSGI apps and middleware.
- **Talks to** SQLite, PostgreSQL, MySQL, MSSQL, FireBird, Oracle, IBM DB2, Informix, Ingres, and Google App Engine. 
+- **Runs on** Windows, Mac, Unix/Linux, Google App Engine, Amazon EC2, and almost any web hosting via Python 2.7/3.5/3.6/pypy.
+- **Runs with** Apache, Nginx, Lighttpd, Cherokee and almost any other web server via CGI, FastCGI, WSGI, mod_proxy, and/or mod_python. It can embed third party WSGI apps and middleware.
+- **Talks to** SQLite, PostgreSQL, MySQL, MSSQL, FireBird, Sybase, Oracle, IBM DB2, Informix, Ingres, MongoDB, and Google App Engine. 
 - **Secure** [[It prevents the most common types of vulnerabilities http://web2py.com/examples/default/security]] including Cross Site Scripting, Injection Flaws, and Malicious File Execution.
 - **Enforces good Software Engineering practices** (Model-View-Controller design, Server-side form validation, postbacks) that make the code more readable, scalable, and maintainable.
 - **Speaks multiple protocols** HTML/XML, RSS/ATOM, RTF, PDF, JSON, AJAX, XML-RPC, CSV, REST, WIKI, Flash/AMF, and Linked Data (RDF). 
@@ -12,6 +12,10 @@
        $.error('web2py.js has already been loaded!');
    }

+    var FORMDATA_IS_SUPPORTED = typeof(FormData) !== 'undefined';
+    var animateIn = 'fadeIn';
+    // var animateIn = 'slideDown';
+
    String.prototype.reverse = function () {
        return this.split('').reverse().join('');
    };
@@ -178,7 +182,7 @@
        },
        /* manage errors in forms */
        manage_errors: function (target) {
-            $('div.error', target).hide().slideDown('slow');
+            $('div.error', target).hide()[animateIn]('slow');
        },
        after_ajax: function (xhr) {
            /* called whenever an ajax request completes */
@@ -263,13 +267,17 @@
                }
            });
            /* help preventing double form submission for normal form (not LOADed) */
-            $(doc).on('submit', 'form', function () {
-                var submit_button = $(this).find(web2py.formInputClickSelector);
-                web2py.disableElement(submit_button);
+            $(doc).on('submit', 'form', function (e) {
+                var submit_buttons = $(this).find(web2py.formInputClickSelector);
+                submit_buttons.each(function() {
+                    web2py.disableElement($(this));
+                })
                /* safeguard in case the form doesn't trigger a refresh,
                see https://github.com/web2py/web2py/issues/1100 */
                setTimeout(function () {
-                    web2py.enableElement(submit_button);
+                    submit_buttons.each(function() {
+                        web2py.enableElement($(this));
+                    });
                }, 5000);
            });
            doc.ajaxSuccess(function (e, xhr) {
@@ -320,7 +328,15 @@
                form.submit(function (e) {
                    web2py.disableElement(form.find(web2py.formInputClickSelector));
                    web2py.hide_flash();
-                    web2py.ajax_page('post', url, form.serialize(), target, form);
+
+                    var formData;
+                    if (FORMDATA_IS_SUPPORTED) {
+                        formData = new FormData(form[0]); // Allows file uploads.
+                    } else {
+                        formData = form.serialize(); // Fallback for older browsers.
+                    }
+                    web2py.ajax_page('post', url, formData, target, form);
+
                    e.preventDefault();
                });
                form.on('click', web2py.formInputClickSelector, function (e) {
@@ -339,11 +355,18 @@
            if (web2py.isUndefined(element)) element = $(document);
            /* if target is not there, fill it with something that there isn't in the page*/
            if (web2py.isUndefined(target) || target === '') target = 'w2p_none';
+
+            /* processData and contentType must be set to false when passing a FormData
+               object to jQuery.ajax. */
+            var isFormData = Object.prototype.toString.call(data) === '[object FormData]';
+            var contentType = isFormData ? false : 'application/x-www-form-urlencoded; charset=UTF-8';
            if (web2py.fire(element, 'ajax:before', null, target)) { /*test a usecase, should stop here if returns false */
                $.ajax({
                    'type': method,
                    'url': action,
                    'data': data,
+                    'processData': !isFormData,
+                    'contentType': contentType,
                    'beforeSend': function (xhr, settings) {
                        xhr.setRequestHeader('web2py-component-location', document.location);
                        xhr.setRequestHeader('web2py-component-element', target);
@@ -595,7 +618,7 @@
            var flash = $('.w2p_flash');
            web2py.hide_flash();
            flash.html(message).addClass(status);
-            if (flash.html()) flash.append('<span id="closeflash"> &times; </span>').slideDown();
+            if (flash.html()) flash.append('<span id="closeflash"> &times; </span>')[animateIn]();
        },
        hide_flash: function () {
            $('.w2p_flash').fadeOut(0).html('');
@@ -609,7 +632,7 @@
                for (var k = 0; k < triggers[id].length; k++) {
                    var dep = $('#' + triggers[id][k], target);
                    var tr = $('#' + triggers[id][k] + '__row', target);
-                    if (t.is(dep.attr('data-show-if'))) tr.slideDown();
+                    if (t.is(dep.attr('data-show-if'))) tr[animateIn]();
                    else tr.hide();
                }
            };
@@ -699,8 +722,9 @@
            });
        },
        /* Disables form elements:
+        - Does not disable elements with 'data-w2p_disable' attribute
        - Caches element value in 'w2p_enable_with' data store
-        - Replaces element text with value of 'data-disable-with' attribute
+        - Replaces element text with value of 'data-w2p_disable_with' attribute
        - Sets disabled property to true
        */
        disableFormElements: function (form) {
@@ -712,13 +736,15 @@
                if (!web2py.isUndefined(disable)) {
                    return false;
                }
-                if (web2py.isUndefined(disable_with)) {
-                    element.data('w2p_disable_with', element[method]());
+                if (!element.is(':file')) { // Altering file input values is not allowed.
+                    if (web2py.isUndefined(disable_with)) {
+                        element.data('w2p_disable_with', element[method]());
+                    }
+                    if (web2py.isUndefined(element.data('w2p_enable_with'))) {
+                        element.data('w2p_enable_with', element[method]());
+                    }
+                    element[method](element.data('w2p_disable_with'));
                }
-                if (web2py.isUndefined(element.data('w2p_enable_with'))) {
-                    element.data('w2p_enable_with', element[method]());
-                }
-                element[method](element.data('w2p_disable_with'));
                element.prop('disabled', true);
            });
        },
@@ -799,4 +825,4 @@ web2py_event_handlers = jQuery.web2py.event_handlers;
 web2py_trap_link = jQuery.web2py.trap_link;
 web2py_calc_entropy = jQuery.web2py.calc_entropy;
 */
-/* compatibility code - end*/
+/* compatibility code - end*/
@@ -239,7 +239,7 @@
    {{=T("No databases in this application")}}
  {{else:}}    
    <div id="vis"></div>
-      <link rel="stylesheet" href="{{=URL('static','css/d3_graph.css')}}"/>
+      <link rel="stylesheet" href="{{=URL('admin','static','css/d3_graph.css')}}"/>
      <script>
        // Define the d3 input data
        {{from gluon.serializers import json }}
@@ -48,7 +48,7 @@
      </tr>
      <tr>
        <td>
-          <a class="btn btn180 rounded green" href="https://dl.dropbox.com/u/18065445/web2py/web2py_manual_5th.pdf">Manual</a>
+          <a class="btn btn180 rounded green" href="http://mdipierro.github.io/web2py/web2py_manual_5th.pdf">Manual</a>
        </td>
        <td>
          <a class="btn btn180 rounded" href="https://github.com/web2py/web2py/releases">Change Log</a>
@@ -657,37 +657,36 @@ def d3_graph_model():
    Create a list of table dicts, called "nodes"
    """
    
-    data = {}
    nodes = []
    links = []

-    subgraphs = dict()
+    for database in databases:
+        db = eval_in_global_env(database)
+        for tablename in db.tables:
+            fields = []
+            for field in db[tablename]:
+                f_type = field.type
+                if not isinstance(f_type,str):
+                    disp = ' '
+                elif f_type == 'string':
+                    disp =  field.length
+                elif f_type == 'id':
+                    disp =  "PK"
+                elif f_type.startswith('reference') or \
+                    f_type.startswith('list:reference'):
+                    disp = "FK"
+                else:
+                    disp = ' '
+                fields.append(dict(name= field.name, type=field.type, disp = disp))

-    for tablename in db.tables:
-        fields = []
-        for field in db[tablename]:
-            f_type = field.type
-            if not isinstance(f_type,str):
-                disp = ' '
-            elif f_type == 'string':
-                disp =  field.length
-            elif f_type == 'id':
-                disp =  "PK"
-            elif f_type.startswith('reference') or \
-                f_type.startswith('list:reference'):
-                disp = "FK"
-            else:
-                disp = ' '
-            fields.append(dict(name= field.name, type=field.type, disp = disp))
+                if isinstance(f_type,str) and (
+                    f_type.startswith('reference') or
+                    f_type.startswith('list:reference')):
+                    referenced_table = f_type.split()[1].split('.')[0]

-            if isinstance(f_type,str) and (
-                f_type.startswith('reference') or
-                f_type.startswith('list:reference')):
-                referenced_table = f_type.split()[1].split('.')[0]
+                    links.append(dict(source=tablename, target = referenced_table))

-                links.append(dict(source=tablename, target = referenced_table))
-
-        nodes.append(dict(name=tablename, type="table", fields = fields))
+            nodes.append(dict(name=tablename, type="table", fields = fields))

    # d3 v4 allows individual modules to be specified.  The complete d3 library is included below.
    response.files.append(URL('admin','static','js/d3.min.js'))
@@ -1,26 +1,35 @@
 # -*- coding: utf-8 -*-
-# this file is released under public domain and you can use without limitations
-
 # -------------------------------------------------------------------------
 # This is a sample controller
-# - index is the default action of any application
-# - user is required for authentication and authorization
-# - download is for downloading files uploaded in the db (does streaming)
+# this file is released under public domain and you can use without limitations
 # -------------------------------------------------------------------------

-
+# ---- example index page ----
 def index():
-    """
-    example action using the internationalization operator T and flash
-    rendered by views/default/index.html or views/generic.html
-
-    if you need a simple wiki simply replace the two lines below with:
-    return auth.wiki()
-    """
    response.flash = T("Hello World")
    return dict(message=T('Welcome to web2py!'))

+# ---- API (example) -----
+@auth.requires_login()
+def api_get_user_email():
+    if not request.env.request_method == 'GET': raise HTTP(403)
+    return response.json({'status':'success', 'email':auth.user.email})

+# ---- Smart Grid (example) -----
+@auth.requires_membership('admin') # can only be accessed by members of admin groupd
+def grid():
+    response.view = 'generic.html' # use a generic view
+    tablename = request.args(0)
+    if not tablename in db.tables: raise HTTP(403)
+    grid = SQLFORM.smartgrid(db[tablename], args=[tablename], deletable=False, editable=False)
+    return dict(grid=grid)
+
+# ---- Embedded wiki (example) ----
+def wiki():
+    auth.wikimenu() # add the wiki to the menu
+    return auth.wiki() 
+
+# ---- Action for login/register/etc (required for auth) -----
 def user():
    """
    exposes:
@@ -39,7 +48,7 @@ def user():
    """
    return dict(form=auth())

-
+# ---- action to server uploaded static content (required) ---
@cache.action()
 def download():
    """
@@ -47,15 +56,3 @@ def download():
    http://..../[app]/default/download/[filename]
    """
    return response.download(request, db)
-
-
-def call():
-    """
-    exposes services. for example:
-    http://..../[app]/default/call/jsonrpc
-    decorate with @services.jsonrpc the functions to expose
-    supports xml, json, xmlrpc, jsonrpc, amfrpc, rss, csv
-    """
-    return service()
-
-
@@ -1,12 +1,19 @@
 # -*- coding: utf-8 -*-

+# -------------------------------------------------------------------------
+# AppConfig configuration made easy. Look inside private/appconfig.ini
+# Auth is for authenticaiton and access control
+# -------------------------------------------------------------------------
+from gluon.contrib.appconfig import AppConfig
+from gluon.tools import Auth
+
 # -------------------------------------------------------------------------
 # This scaffolding model makes your app work on Google App Engine too
 # File is released under public domain and you can use without limitations
 # -------------------------------------------------------------------------

-if request.global_settings.web2py_version < "2.14.1":
-    raise HTTP(500, "Requires web2py 2.13.3 or newer")
+if request.global_settings.web2py_version < "2.15.5":
+    raise HTTP(500, "Requires web2py 2.15.5 or newer")

 # -------------------------------------------------------------------------
 # if SSL/HTTPS is properly configured and you want all HTTP requests to
@@ -14,23 +21,18 @@ if request.global_settings.web2py_version < "2.14.1":
 # -------------------------------------------------------------------------
 # request.requires_https()

-# -------------------------------------------------------------------------
-# app configuration made easy. Look inside private/appconfig.ini
-# -------------------------------------------------------------------------
-from gluon.contrib.appconfig import AppConfig
-
 # -------------------------------------------------------------------------
 # once in production, remove reload=True to gain full speed
 # -------------------------------------------------------------------------
-myconf = AppConfig(reload=True)
+configuration = AppConfig(reload=True)

 if not request.env.web2py_runtime_gae:
    # ---------------------------------------------------------------------
    # if NOT running on Google App Engine use SQLite or other DB
    # ---------------------------------------------------------------------
-    db = DAL(myconf.get('db.uri'),
-             pool_size=myconf.get('db.pool_size'),
-             migrate_enabled=myconf.get('db.migrate'),
+    db = DAL(configuration.get('db.uri'),
+             pool_size=configuration.get('db.pool_size'),
+             migrate_enabled=configuration.get('db.migrate'),
             check_reserved=['all'])
 else:
    # ---------------------------------------------------------------------
@@ -52,12 +54,15 @@ else:
 # by default give a view/generic.extension to all actions from localhost
 # none otherwise. a pattern can be 'controller/function.extension'
 # -------------------------------------------------------------------------
-response.generic_patterns = ['*'] if request.is_local else []
+response.generic_patterns = [] 
+if request.is_local and not configuration.get('app.production'):
+    response.generic_patterns.append('*')
+
 # -------------------------------------------------------------------------
 # choose a style for forms
 # -------------------------------------------------------------------------
-response.formstyle = myconf.get('forms.formstyle')  # or 'bootstrap3_stacked' or 'bootstrap2' or other
-response.form_label_separator = myconf.get('forms.separator') or ''
+response.formstyle = 'bootstrap4_inline'
+response.form_label_separator = ''

 # -------------------------------------------------------------------------
 # (optional) optimize handling of static files
@@ -80,27 +85,24 @@ response.form_label_separator = myconf.get('forms.separator') or ''
 # (more options discussed in gluon/tools.py)
 # -------------------------------------------------------------------------

-from gluon.tools import Auth, Service, PluginManager
-
 # host names must be a list of allowed host names (glob syntax allowed)
-auth = Auth(db, host_names=myconf.get('host.names'))
-service = Service()
-plugins = PluginManager()
+auth = Auth(db, host_names=configuration.get('host.names'))

 # -------------------------------------------------------------------------
-# create all tables needed by auth if not custom tables
+# create all tables needed by auth, maybe add a list of extra fields
 # -------------------------------------------------------------------------
+auth.settings.extra_fields['auth_user'] = []
 auth.define_tables(username=False, signature=False)

 # -------------------------------------------------------------------------
 # configure email
 # -------------------------------------------------------------------------
 mail = auth.settings.mailer
-mail.settings.server = 'logging' if request.is_local else myconf.get('smtp.server')
-mail.settings.sender = myconf.get('smtp.sender')
-mail.settings.login = myconf.get('smtp.login')
-mail.settings.tls = myconf.get('smtp.tls') or False
-mail.settings.ssl = myconf.get('smtp.ssl') or False
+mail.settings.server = 'logging' if request.is_local else configuration.get('smtp.server')
+mail.settings.sender = configuration.get('smtp.sender')
+mail.settings.login = configuration.get('smtp.login')
+mail.settings.tls = configuration.get('smtp.tls') or False
+mail.settings.ssl = configuration.get('smtp.ssl') or False

 # -------------------------------------------------------------------------
 # configure auth policy
@@ -109,6 +111,27 @@ auth.settings.registration_requires_verification = False
 auth.settings.registration_requires_approval = False
 auth.settings.reset_password_requires_verification = True

+# -------------------------------------------------------------------------  
+# read more at http://dev.w3.org/html5/markup/meta.name.html               
+# -------------------------------------------------------------------------
+response.meta.author = configuration.get('app.author')
+response.meta.description = configuration.get('app.description')
+response.meta.keywords = configuration.get('app.keywords')
+response.meta.generator = configuration.get('app.generator')
+response.show_toolbar = configuration.get('app.toolbar')
+
+# -------------------------------------------------------------------------
+# your http://google.com/analytics id                                      
+# -------------------------------------------------------------------------
+response.google_analytics_id = configuration.get('google.analytics_id')
+
+# -------------------------------------------------------------------------
+# maybe use the scheduler
+# -------------------------------------------------------------------------
+if configuration.get('scheduler.enabled'):
+    from gluon.scheduler import Scheduler
+    scheduler = Scheduler(db, heartbeat=configure.get('heartbeat'))
+
 # -------------------------------------------------------------------------
 # Define your tables below (or better in another model file) for example
 #
@@ -1,29 +1,6 @@
 # -*- coding: utf-8 -*-
 # this file is released under public domain and you can use without limitations

-# ----------------------------------------------------------------------------------------------------------------------
-# Customize your APP title, subtitle and menus here
-# ----------------------------------------------------------------------------------------------------------------------
-
-response.logo = A(B('web', SPAN(2), 'py'), XML('&trade;&nbsp;'),
-                  _class="navbar-brand", _href="http://www.web2py.com/",
-                  _id="web2py-logo")
-response.title = request.application.replace('_', ' ').title()
-response.subtitle = ''
-
-# ----------------------------------------------------------------------------------------------------------------------
-# read more at http://dev.w3.org/html5/markup/meta.name.html
-# ----------------------------------------------------------------------------------------------------------------------
-response.meta.author = myconf.get('app.author')
-response.meta.description = myconf.get('app.description')
-response.meta.keywords = myconf.get('app.keywords')
-response.meta.generator = myconf.get('app.generator')
-
-# ----------------------------------------------------------------------------------------------------------------------
-# your http://google.com/analytics id
-# ----------------------------------------------------------------------------------------------------------------------
-response.google_analytics_id = None
-
 # ----------------------------------------------------------------------------------------------------------------------
 # this is the main application menu add/remove items as required
 # ----------------------------------------------------------------------------------------------------------------------
@@ -32,53 +9,42 @@ response.menu = [
    (T('Home'), False, URL('default', 'index'), [])
 ]

-DEVELOPMENT_MENU = True
-
-
 # ----------------------------------------------------------------------------------------------------------------------
-# provide shortcuts for development. remove in production
+# provide shortcuts for development. you can remove everything below in production
 # ----------------------------------------------------------------------------------------------------------------------

-def _():
-    # ------------------------------------------------------------------------------------------------------------------
-    # shortcuts
-    # ------------------------------------------------------------------------------------------------------------------
-    app = request.application
-    ctr = request.controller
-    # ------------------------------------------------------------------------------------------------------------------
-    # useful links to internal and external resources
-    # ------------------------------------------------------------------------------------------------------------------
+if not configuration.get('app.production'):
+    _app = request.application
    response.menu += [
        (T('My Sites'), False, URL('admin', 'default', 'site')),
        (T('This App'), False, '#', [
-            (T('Design'), False, URL('admin', 'default', 'design/%s' % app)),
-            LI(_class="divider"),
+            (T('Design'), False, URL('admin', 'default', 'design/%s' % _app)),
            (T('Controller'), False,
             URL(
-                 'admin', 'default', 'edit/%s/controllers/%s.py' % (app, ctr))),
+                 'admin', 'default', 'edit/%s/controllers/%s.py' % (_app, request.controller))),
            (T('View'), False,
             URL(
-                 'admin', 'default', 'edit/%s/views/%s' % (app, response.view))),
+                 'admin', 'default', 'edit/%s/views/%s' % (_app, response.view))),
            (T('DB Model'), False,
             URL(
-                 'admin', 'default', 'edit/%s/models/db.py' % app)),
+                 'admin', 'default', 'edit/%s/models/db.py' % _app)),
            (T('Menu Model'), False,
             URL(
-                 'admin', 'default', 'edit/%s/models/menu.py' % app)),
+                 'admin', 'default', 'edit/%s/models/menu.py' % _app)),
            (T('Config.ini'), False,
             URL(
-                 'admin', 'default', 'edit/%s/private/appconfig.ini' % app)),
+                 'admin', 'default', 'edit/%s/private/appconfig.ini' % _app)),
            (T('Layout'), False,
             URL(
-                 'admin', 'default', 'edit/%s/views/layout.html' % app)),
+                 'admin', 'default', 'edit/%s/views/layout.html' % _app)),
            (T('Stylesheet'), False,
             URL(
-                 'admin', 'default', 'edit/%s/static/css/web2py-bootstrap3.css' % app)),
-            (T('Database'), False, URL(app, 'appadmin', 'index')),
+                 'admin', 'default', 'edit/%s/static/css/web2py-bootstrap3.css' % _app)),
+            (T('Database'), False, URL(_app, 'appadmin', 'index')),
            (T('Errors'), False, URL(
-                'admin', 'default', 'errors/' + app)),
+                'admin', 'default', 'errors/' + _app)),
            (T('About'), False, URL(
-                'admin', 'default', 'about/' + app)),
+                'admin', 'default', 'about/' + _app)),
        ]),
        ('web2py.com', False, '#', [
            (T('Download'), False,
@@ -98,7 +64,6 @@ def _():
        ]),
        (T('Documentation'), False, '#', [
            (T('Online book'), False, 'http://www.web2py.com/book'),
-            LI(_class="divider"),
            (T('Preface'), False,
             'http://www.web2py.com/book/default/chapter/00'),
            (T('Introduction'), False,
@@ -143,9 +108,3 @@ def _():
        ]),
    ]

-
-if DEVELOPMENT_MENU:
-    _()
-
-if "auth" in locals():
-    auth.wikimenu()
@@ -5,6 +5,8 @@ author      = Your Name <you@example.com>
 description = a cool new app
 keywords    = web2py, python, framework
 generator   = Web2py Web Framework
+production  = false
+toolbar     = false

 ; Host configuration
 [host]
@@ -14,7 +16,6 @@ names = localhost:*, 127.0.0.1:*, *:*, *
 [db]
 uri       = sqlite://storage.sqlite
 migrate   = true
-; ignored for sqlite
 pool_size = 10  

 ; smtp address and credentials
@@ -25,7 +26,9 @@ login  = username:password
 tls    = true
 ssl    = true

-; form styling
-[forms]
-formstyle = bootstrap3_inline
-separator = 
+[scheduler]
+enabled   = false
+heartbeat = 1
+
+[google]
+analytics_id =
@@ -20,8 +20,8 @@
 # YOU CAN COPY THIS FILE TO ANY APPLICATION'S ROOT DIRECTORY WITHOUT CHANGES!
 # ----------------------------------------------------------------------------------------------------------------------

-from fileutils import abspath
-from languages import read_possible_languages
+from gluon.fileutils import abspath
+from gluon.languages import read_possible_languages

 possible_languages = read_possible_languages(abspath('applications', app))
 # ----------------------------------------------------------------------------------------------------------------------
@@ -108,7 +108,6 @@ select.autocomplete {
    background: url(../images/background.jpg) no-repeat center center;
 }
 body {
-    padding-top: 60px;
    margin-bottom: 60px;
 }
 header {
@@ -126,8 +125,7 @@ html {
    bottom: 0;
    width: 100%;
    height: 60px;
-    background: #333;
-    color: #aaa;
+    background: #f7f7f7;
 }
 header h1 {
    color: #FFF!important;
@@ -1,6 +0,0 @@
-/*! Respond.js v1.4.2: min/max-width media query polyfill
- * Copyright 2014 Scott Jehl
- * Licensed under MIT
- * http://j.mp/respondjs */
-
-!function(a){"use strict";a.matchMedia=a.matchMedia||function(a){var b,c=a.documentElement,d=c.firstElementChild||c.firstChild,e=a.createElement("body"),f=a.createElement("div");return f.id="mq-test-1",f.style.cssText="position:absolute;top:-100em",e.style.background="none",e.appendChild(f),function(a){return f.innerHTML='&shy;<style media="'+a+'"> #mq-test-1 { width: 42px; }</style>',c.insertBefore(e,d),b=42===f.offsetWidth,c.removeChild(e),{matches:b,media:a}}}(a.document)}(this),function(a){"use strict";function b(){v(!0)}var c={};a.respond=c,c.update=function(){};var d=[],e=function(){var b=!1;try{b=new a.XMLHttpRequest}catch(c){b=new a.ActiveXObject("Microsoft.XMLHTTP")}return function(){return b}}(),f=function(a,b){var c=e();c&&(c.open("GET",a,!0),c.onreadystatechange=function(){4!==c.readyState||200!==c.status&&304!==c.status||b(c.responseText)},4!==c.readyState&&c.send(null))},g=function(a){return a.replace(c.regex.minmaxwh,"").match(c.regex.other)};if(c.ajax=f,c.queue=d,c.unsupportedmq=g,c.regex={media:/@media[^\{]+\{([^\{\}]*\{[^\}\{]*\})+/gi,keyframes:/@(?:\-(?:o|moz|webkit)\-)?keyframes[^\{]+\{(?:[^\{\}]*\{[^\}\{]*\})+[^\}]*\}/gi,comments:/\/\*[^*]*\*+([^/][^*]*\*+)*\//gi,urls:/(url\()['"]?([^\/\)'"][^:\)'"]+)['"]?(\))/g,findStyles:/@media *([^\{]+)\{([\S\s]+?)$/,only:/(only\s+)?([a-zA-Z]+)\s?/,minw:/\(\s*min\-width\s*:\s*(\s*[0-9\.]+)(px|em)\s*\)/,maxw:/\(\s*max\-width\s*:\s*(\s*[0-9\.]+)(px|em)\s*\)/,minmaxwh:/\(\s*m(in|ax)\-(height|width)\s*:\s*(\s*[0-9\.]+)(px|em)\s*\)/gi,other:/\([^\)]*\)/g},c.mediaQueriesSupported=a.matchMedia&&null!==a.matchMedia("only all")&&a.matchMedia("only all").matches,!c.mediaQueriesSupported){var h,i,j,k=a.document,l=k.documentElement,m=[],n=[],o=[],p={},q=30,r=k.getElementsByTagName("head")[0]||l,s=k.getElementsByTagName("base")[0],t=r.getElementsByTagName("link"),u=function(){var a,b=k.createElement("div"),c=k.body,d=l.style.fontSize,e=c&&c.style.fontSize,f=!1;return b.style.cssText="position:absolute;font-size:1em;width:1em",c||(c=f=k.createElement("body"),c.style.background="none"),l.style.fontSize="100%",c.style.fontSize="100%",c.appendChild(b),f&&l.insertBefore(c,l.firstChild),a=b.offsetWidth,f?l.removeChild(c):c.removeChild(b),l.style.fontSize=d,e&&(c.style.fontSize=e),a=j=parseFloat(a)},v=function(b){var c="clientWidth",d=l[c],e="CSS1Compat"===k.compatMode&&d||k.body[c]||d,f={},g=t[t.length-1],p=(new Date).getTime();if(b&&h&&q>p-h)return a.clearTimeout(i),i=a.setTimeout(v,q),void 0;h=p;for(var s in m)if(m.hasOwnProperty(s)){var w=m[s],x=w.minw,y=w.maxw,z=null===x,A=null===y,B="em";x&&(x=parseFloat(x)*(x.indexOf(B)>-1?j||u():1)),y&&(y=parseFloat(y)*(y.indexOf(B)>-1?j||u():1)),w.hasquery&&(z&&A||!(z||e>=x)||!(A||y>=e))||(f[w.media]||(f[w.media]=[]),f[w.media].push(n[w.rules]))}for(var C in o)o.hasOwnProperty(C)&&o[C]&&o[C].parentNode===r&&r.removeChild(o[C]);o.length=0;for(var D in f)if(f.hasOwnProperty(D)){var E=k.createElement("style"),F=f[D].join("\n");E.type="text/css",E.media=D,r.insertBefore(E,g.nextSibling),E.styleSheet?E.styleSheet.cssText=F:E.appendChild(k.createTextNode(F)),o.push(E)}},w=function(a,b,d){var e=a.replace(c.regex.comments,"").replace(c.regex.keyframes,"").match(c.regex.media),f=e&&e.length||0;b=b.substring(0,b.lastIndexOf("/"));var h=function(a){return a.replace(c.regex.urls,"$1"+b+"$2$3")},i=!f&&d;b.length&&(b+="/"),i&&(f=1);for(var j=0;f>j;j++){var k,l,o,p;i?(k=d,n.push(h(a))):(k=e[j].match(c.regex.findStyles)&&RegExp.$1,n.push(RegExp.$2&&h(RegExp.$2))),o=k.split(","),p=o.length;for(var q=0;p>q;q++)l=o[q],g(l)||m.push({media:l.split("(")[0].match(c.regex.only)&&RegExp.$2||"all",rules:n.length-1,hasquery:l.indexOf("(")>-1,minw:l.match(c.regex.minw)&&parseFloat(RegExp.$1)+(RegExp.$2||""),maxw:l.match(c.regex.maxw)&&parseFloat(RegExp.$1)+(RegExp.$2||"")})}v()},x=function(){if(d.length){var b=d.shift();f(b.href,function(c){w(c,b.href,b.media),p[b.href]=!0,a.setTimeout(function(){x()},0)})}},y=function(){for(var b=0;b<t.length;b++){var c=t[b],e=c.href,f=c.media,g=c.rel&&"stylesheet"===c.rel.toLowerCase();e&&g&&!p[e]&&(c.styleSheet&&c.styleSheet.rawCssText?(w(c.styleSheet.rawCssText,e,f),p[e]=!0):(!/^([a-zA-Z:]*\/\/)/.test(e)&&!s||e.replace(RegExp.$1,"").split("/")[0]===a.location.host)&&("//"===e.substring(0,2)&&(e=a.location.protocol+e),d.push({href:e,media:f})))}x()};y(),c.update=y,c.getEmValue=u,a.addEventListener?a.addEventListener("resize",b,!1):a.attachEvent&&a.attachEvent("onresize",b)}}(this);
@@ -12,6 +12,10 @@
        $.error('web2py.js has already been loaded!');
    }

+    var FORMDATA_IS_SUPPORTED = typeof(FormData) !== 'undefined';
+    var animateIn = 'fadeIn';
+    // var animateIn = 'slideDown';
+
    String.prototype.reverse = function () {
        return this.split('').reverse().join('');
    };
@@ -178,7 +182,7 @@
        },
        /* manage errors in forms */
        manage_errors: function (target) {
-            $('div.error', target).hide().slideDown('slow');
+            $('div.error', target).hide()[animateIn]('slow');
        },
        after_ajax: function (xhr) {
            /* called whenever an ajax request completes */
@@ -263,13 +267,17 @@
                }
            });
            /* help preventing double form submission for normal form (not LOADed) */
-            $(doc).on('submit', 'form', function () {
-                var submit_button = $(this).find(web2py.formInputClickSelector);
-                web2py.disableElement(submit_button);
+            $(doc).on('submit', 'form', function (e) {
+                var submit_buttons = $(this).find(web2py.formInputClickSelector);
+                submit_buttons.each(function() {
+                    web2py.disableElement($(this));
+                })
                /* safeguard in case the form doesn't trigger a refresh,
                see https://github.com/web2py/web2py/issues/1100 */
                setTimeout(function () {
-                    web2py.enableElement(submit_button);
+                    submit_buttons.each(function() {
+                        web2py.enableElement($(this));
+                    });
                }, 5000);
            });
            doc.ajaxSuccess(function (e, xhr) {
@@ -320,7 +328,15 @@
                form.submit(function (e) {
                    web2py.disableElement(form.find(web2py.formInputClickSelector));
                    web2py.hide_flash();
-                    web2py.ajax_page('post', url, form.serialize(), target, form);
+
+                    var formData;
+                    if (FORMDATA_IS_SUPPORTED) {
+                        formData = new FormData(form[0]); // Allows file uploads.
+                    } else {
+                        formData = form.serialize(); // Fallback for older browsers.
+                    }
+                    web2py.ajax_page('post', url, formData, target, form);
+
                    e.preventDefault();
                });
                form.on('click', web2py.formInputClickSelector, function (e) {
@@ -339,11 +355,18 @@
            if (web2py.isUndefined(element)) element = $(document);
            /* if target is not there, fill it with something that there isn't in the page*/
            if (web2py.isUndefined(target) || target === '') target = 'w2p_none';
+
+            /* processData and contentType must be set to false when passing a FormData
+               object to jQuery.ajax. */
+            var isFormData = Object.prototype.toString.call(data) === '[object FormData]';
+            var contentType = isFormData ? false : 'application/x-www-form-urlencoded; charset=UTF-8';
            if (web2py.fire(element, 'ajax:before', null, target)) { /*test a usecase, should stop here if returns false */
                $.ajax({
                    'type': method,
                    'url': action,
                    'data': data,
+                    'processData': !isFormData,
+                    'contentType': contentType,
                    'beforeSend': function (xhr, settings) {
                        xhr.setRequestHeader('web2py-component-location', document.location);
                        xhr.setRequestHeader('web2py-component-element', target);
@@ -595,7 +618,7 @@
            var flash = $('.w2p_flash');
            web2py.hide_flash();
            flash.html(message).addClass(status);
-            if (flash.html()) flash.append('<span id="closeflash"> &times; </span>').slideDown();
+            if (flash.html()) flash.append('<span id="closeflash"> &times; </span>')[animateIn]();
        },
        hide_flash: function () {
            $('.w2p_flash').fadeOut(0).html('');
@@ -609,7 +632,7 @@
                for (var k = 0; k < triggers[id].length; k++) {
                    var dep = $('#' + triggers[id][k], target);
                    var tr = $('#' + triggers[id][k] + '__row', target);
-                    if (t.is(dep.attr('data-show-if'))) tr.slideDown();
+                    if (t.is(dep.attr('data-show-if'))) tr[animateIn]();
                    else tr.hide();
                }
            };
@@ -699,8 +722,9 @@
            });
        },
        /* Disables form elements:
+        - Does not disable elements with 'data-w2p_disable' attribute
        - Caches element value in 'w2p_enable_with' data store
-        - Replaces element text with value of 'data-disable-with' attribute
+        - Replaces element text with value of 'data-w2p_disable_with' attribute
        - Sets disabled property to true
        */
        disableFormElements: function (form) {
@@ -712,13 +736,15 @@
                if (!web2py.isUndefined(disable)) {
                    return false;
                }
-                if (web2py.isUndefined(disable_with)) {
-                    element.data('w2p_disable_with', element[method]());
+                if (!element.is(':file')) { // Altering file input values is not allowed.
+                    if (web2py.isUndefined(disable_with)) {
+                        element.data('w2p_disable_with', element[method]());
+                    }
+                    if (web2py.isUndefined(element.data('w2p_enable_with'))) {
+                        element.data('w2p_enable_with', element[method]());
+                    }
+                    element[method](element.data('w2p_disable_with'));
                }
-                if (web2py.isUndefined(element.data('w2p_enable_with'))) {
-                    element.data('w2p_enable_with', element[method]());
-                }
-                element[method](element.data('w2p_disable_with'));
                element.prop('disabled', true);
            });
        },
@@ -799,4 +825,4 @@ web2py_event_handlers = jQuery.web2py.event_handlers;
 web2py_trap_link = jQuery.web2py.trap_link;
 web2py_calc_entropy = jQuery.web2py.calc_entropy;
 */
-/* compatibility code - end*/
+/* compatibility code - end*/
@@ -239,7 +239,7 @@
    {{=T("No databases in this application")}}
  {{else:}}    
    <div id="vis"></div>
-      <link rel="stylesheet" href="{{=URL('static','css/d3_graph.css')}}"/>
+      <link rel="stylesheet" href="{{=URL('admin','static','css/d3_graph.css')}}"/>
      <script>
        // Define the d3 input data
        {{from gluon.serializers import json }}
@@ -1,15 +1,9 @@
-{{left_sidebar_enabled,right_sidebar_enabled=False,('message' in globals())}}
 {{extend 'layout.html'}}

 {{block header}}
-    <header class="container-fluid background">
-      <div class="jumbotron text-center">
-        {{if response.title:}}
-        <h1>{{=response.title}}
-          <small>{{=response.subtitle or ''}}</small></h1>
-        {{pass}}
-      </div>
-    </header>
+<center style="background-color: #333; color:white; padding:30px">
+  <h1>/{{=request.application}}/{{=request.controller}}/{{=request.function}}
+</center>
 {{end}}

 {{if 'message' in globals():}}
@@ -27,26 +21,23 @@
    _href=URL('admin','default','peek',args=(request.application,'views',request.controller,'index.html')))))}}</li>
  <li>{{=T('You can modify this application and adapt it to your needs')}}</li>
 </ol>
+<center style="padding:50px">
+  <a class="btn btn-primary" href="{{=URL('admin','default','index')}}">
+    <i class="glyphicon glyphicon-cog"></i>
+    {{=T("admin")}}
+  </a>
+  <a class="btn btn-secondary" href="{{=URL('examples','default','index')}}">{{=T("Online examples")}}</a>
+  <a class="btn btn-secondary" href="http://web2py.com">web2py.com</a>
+  <a class="btn btn-secondary" href="http://web2py.com/book">{{=T('Documentation')}}</a>
+  <a class="btn btn-secondary" href="{{=URL('default','api_get_user_email')}}">{{=T('API Example')}}</a>
+  <a class="btn btn-secondary" href="{{=URL('default','grid/auth_user')}}">{{=T('Grid Example')}}</a>
+  <a class="btn btn-secondary" href="{{=URL('default','wiki')}}">{{=T('Wiki Example')}}</a>
+</center>
 {{elif 'content' in globals():}}
 {{=content}}
 {{else:}}
 {{=BEAUTIFY(response._vars)}}
 {{pass}}

-{{block right_sidebar}}
-<div class="panel panel-info">
-  <div class="panel-heading"><h3 class="panel-title"><a class="btn-block"
-      href="{{=URL('admin','default','index')}}">
-      <i class="glyphicon glyphicon-cog"></i>
-      {{=T("admin")}}
-    </a></h3></div>
-  <div class="panel-body">
-    {{=T("Don't know what to do?")}}
-  </div>
-  <ul class="list-group">
-    <li class="list-group-item">{{=A(T("Online examples"), _href=URL('examples','default','index'))}}</li>
-    <li class="list-group-item"><a href="http://web2py.com">web2py.com</a></li>
-    <li class="list-group-item"><a href="http://web2py.com/book">{{=T('Documentation')}}</a></li>
-  </ul>
-</div>
-{{end}}
+
+
@@ -11,6 +11,3 @@ It is used as default when a view is not provided for your controllers
 {{elif len(response._vars)>1:}}
 {{=BEAUTIFY(response._vars)}}
 {{pass}}
-{{if request.is_local:}}
-{{=response.toolbar()}}
-{{pass}}
@@ -21,84 +21,81 @@
    <meta name="google-site-verification" content="">
    <!-- include stylesheets -->
    <link rel="stylesheet" href="{{=URL('static','css/bootstrap.min.css')}}"/>
-    <link rel="stylesheet" href="{{=URL('static','css/web2py-bootstrap3.css')}}"/>
+    <link rel="stylesheet" href="{{=URL('static','css/web2py-bootstrap4.css')}}"/>
    <link rel="shortcut icon" href="{{=URL('static','images/favicon.ico')}}" type="image/x-icon">
    <link rel="apple-touch-icon" href="{{=URL('static','images/favicon.png')}}">
    <!-- All JavaScript at the bottom, except for Modernizr which enables
         HTML5 elements & feature detects -->
    <script src="{{=URL('static','js/modernizr-2.8.3.min.js')}}"></script>
-    <!--[if lt IE 9]>
-        <script src="{{=URL('static','js/respond-1.4.2.min.js')}}"></script>
-        <![endif]-->
    <!-- Favicons -->
    {{include 'web2py_ajax.html'}} <!-- this includes jquery.js, calendar.js/.css and web2py.js -->
    {{block head}}{{end}}
-    {{
-    # using sidebars need to know what sidebar you want to use
-    mc0 = 'col-md-12'
-    mc1 = 'col-md-9'
-    mc2 = 'col-md-6'
-    left_sidebar_enabled = globals().get('left_sidebar_enabled', False)
-    right_sidebar_enabled = globals().get('right_sidebar_enabled', False)
-    middle_column = {0: mc0, 1: mc1, 2: mc2}[
-    (left_sidebar_enabled and 1 or 0)+(right_sidebar_enabled and 1 or 0)]
-    }}
  </head>
  <body>
-    <!--[if lt IE 8]><p class="browserupgrade">You are using an <strong>outdated</strong> browser. Please <a href="http://browsehappy.com/">upgrade your browser</a> to improve your experience.</p><![endif]-->
    <div class="w2p_flash alert alert-dismissable">{{=response.flash or ''}}</div>
    <!-- Navbar ======================================= -->
-    <nav class="navbar navbar-default navbar-fixed-top" role="navigation">
-      <div class="container-fluid">
-        <div class="navbar-header">
-          <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse">
-            <span class="sr-only">Toggle navigation</span>
-            <span class="icon-bar"></span>
-            <span class="icon-bar"></span>
-            <span class="icon-bar"></span>
-          </button>
-          {{=response.logo or ''}}
-        </div>
-        <div class="collapse navbar-collapse">
-          <ul class="nav navbar-nav navbar-right">
-            {{='auth' in globals() and auth.navbar('Welcome',mode='dropdown') or ''}}
-          </ul>
-          {{if response.menu:}}
-          {{=MENU(response.menu, _class='nav navbar-nav',li_class='dropdown',ul_class='dropdown-menu')}}
+    <nav class="navbar navbar-toggleable-md navbar-light bg-faded">
+      <button class="navbar-toggler navbar-toggler-right" type="button" data-toggle="collapse" data-target="#navbarNavDropdown" aria-controls="navbarNavDropdown" aria-expanded="false" aria-label="Toggle navigation">
+        <span class="navbar-toggler-icon"></span>
+      </button>
+      <a class="navbar-brand" href="http://web2py.com">web2py</a>
+      <div class="collapse navbar-collapse">
+        <ul class="navbar-nav">
+          {{for _item in response.menu or []:}}
+          {{if len(_item)<4 or not _item[3]:}}
+          <li class="nav-item {{if _item[1]:}}active{{pass}}">
+            <a class="nav-link" href="{{=_item[2]}}">{{=_item[0]}}</a>
+          </li>
+          {{else:}}
+          <li class="nav-item dropdown">
+            <a class="nav-link dropdown-toggle" href="{{=_item[2]}}" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">{{=_item[0]}}</a>
+            <div class="dropdown-menu">
+              {{for _subitem in _item[3]:}}
+              <a class="dropdown-item" href="{{=_subitem[2]}}">{{=_subitem[0]}}</a>
+              {{pass}}
+            </div>
+          </li>
          {{pass}}
-        </div>
+          {{pass}}          
+        </ul>
+        <form class="form-inline my-2 my-lg-0">
+          <input class="form-control mr-sm-2" type="text" placeholder="Search">
+        </form>      
+        {{if 'auth' in globals():}}
+        <ul class="navbar-nav navbar-right">
+          <li class="nav-item dropdown">
+            <a class="nav-link dropdown-toggle" href="#" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">
+              {{if auth.user:}}{{=auth.user.first_name}}{{else:}}LOGIN{{pass}}
+            </a>
+            <div class="dropdown-menu">
+              {{if auth.user:}}
+              <a class="dropdown-item" href="{{=URL('default','user/profile')}}">{{=T('Profile')}}</a>
+              <a class="dropdown-item" href="{{=URL('default','user/change_password')}}">{{=T('Change Password')}}</a>
+              <a class="dropdown-item" href="{{=URL('default','user/logout')}}">{{=T('Logout')}}</a>
+              {{else:}}
+              <a class="dropdown-item" href="{{=URL('default','user/login')}}">{{=T('Login')}}</a>
+              <a class="dropdown-item" href="{{=URL('default','user/register')}}">{{=T('Sign up')}}</a>
+              <a class="dropdown-item" href="{{=URL('default','user/request_password')}}">{{=T('Lost Password')}}</a>
+              {{pass}}
+            </div>
+          </li>
+        </ul>        
+        {{pass}}
      </div>
    </nav>
+
    <!-- Masthead ===================================== -->    
    {{block header}}
    {{end}}
    <!-- Main ========================================= -->
    <!-- Begin page content -->
    <div class="container-fluid main-container">
-      {{if left_sidebar_enabled:}}
-        <div class="col-md-3 left-sidebar">
-            {{block left_sidebar}}
-            <h3>Left Sidebar</h3>
-            <p></p>
-            {{end}}
-        </div>
-      {{pass}}
-
-      <div class="{{=middle_column}}">
+      <div class="col-md-12">
        {{block center}}
        {{include}}
        {{end}}
+        {{=response.toolbar() if response.show_toolbar else ''}}
      </div>
-
-      {{if right_sidebar_enabled:}}
-        <div class="col-md-3">
-            {{block right_sidebar}}
-            <h3>Right Sidebar</h3>
-            <p></p>
-            {{end}}
-        </div>
-      {{pass}}
-
    </div>

    {{block footer}} <!-- this is default footer -->      
@@ -114,7 +111,7 @@
    {{end}}
    <!-- The javascript =============================== -->
    <script src="{{=URL('static','js/bootstrap.min.js')}}"></script>
-    <script src="{{=URL('static','js/web2py-bootstrap3.js')}}"></script>
+    <script src="{{=URL('static','js/web2py-bootstrap4.js')}}"></script>
    {{block page_js}}{{end page_js}}
    {{if response.google_analytics_id:}}
    <!-- Analytics ==================================== -->
@@ -35,6 +35,7 @@ if PY2:
    from gluon.contrib import ipaddress
    BytesIO = StringIO
    reduce = reduce
+    reload = reload
    hashlib_md5 = hashlib.md5
    iterkeys = lambda d: d.iterkeys()
    itervalues = lambda d: d.itervalues()
@@ -63,7 +64,7 @@ if PY2:
            return None
        if isinstance(obj, (bytes, bytearray, buffer)):
            return bytes(obj)
-        if isinstance(obj, unicode):
+        if hasattr(obj, 'encode'):
            return obj.encode(charset, errors)
        raise TypeError('Expected bytes')

@@ -77,6 +78,7 @@ else:
    import pickle
    from io import StringIO, BytesIO
    import copyreg
+    from importlib import reload
    from functools import reduce
    from html.parser import HTMLParser
    from http import cookies as Cookie
@@ -122,7 +124,7 @@ else:
            return None
        if isinstance(obj, (bytes, bytearray, memoryview)):
            return bytes(obj)
-        if isinstance(obj, str):
+        if hasattr(obj, 'encode'):
            return obj.encode(charset, errors)
        raise TypeError('Expected bytes')

@@ -151,7 +153,7 @@ def with_metaclass(meta, *bases):
 def to_unicode(obj, charset='utf-8', errors='strict'):
    if obj is None:
        return None
-    if not isinstance(obj, bytes):
+    if not hasattr(obj, 'decode'):
        return text_type(obj)
    return obj.decode(charset, errors)

@@ -54,7 +54,8 @@ def app_pack(app, request, raise_ex=False, filenames=None):

    """
    try:
-        if filenames is None: app_cleanup(app, request)
+        if filenames is None:
+            app_cleanup(app, request)
        filename = apath('../deposit/web2py.app.%s.w2p' % app, request)
        w2p_pack(filename, apath(app, request), filenames=filenames)
        return filename
@@ -104,7 +105,8 @@ def app_cleanup(app, request):
    if os.path.exists(path):
        for f in os.listdir(path):
            try:
-                if f[:1] != '.': os.unlink(os.path.join(path, f))
+                if f[:1] != '.':
+                    os.unlink(os.path.join(path, f))
            except IOError:
                r = False

@@ -113,7 +115,8 @@ def app_cleanup(app, request):
    if os.path.exists(path):
        for f in os.listdir(path):
            try:
-                if f[:1] != '.': recursive_unlink(os.path.join(path, f))
+                if f[:1] != '.':
+                    recursive_unlink(os.path.join(path, f))
            except (OSError, IOError):
                r = False

@@ -123,7 +126,8 @@ def app_cleanup(app, request):
        CacheOnDisk(folder=path).clear()
        for f in os.listdir(path):
            try:
-                if f[:1] != '.': recursive_unlink(os.path.join(path, f))
+                if f[:1] != '.':
+                    recursive_unlink(os.path.join(path, f))
            except (OSError, IOError):
                r = False
    return r
@@ -175,10 +179,9 @@ def app_create(app, request, force=False, key=None, info=False):
            return False
    try:
        w2p_unpack('welcome.w2p', path)
-        for subfolder in [
-            'models', 'views', 'controllers', 'databases',
-            'modules', 'cron', 'errors', 'sessions', 'cache',
-            'languages', 'static', 'private', 'uploads']:
+        for subfolder in ['models', 'views', 'controllers', 'databases',
+                          'modules', 'cron', 'errors', 'sessions', 'cache',
+                          'languages', 'static', 'private', 'uploads']:
            subpath = os.path.join(path, subfolder)
            if not os.path.exists(subpath):
                os.mkdir(subpath)
@@ -368,7 +371,7 @@ def unzip(filename, dir, subfolder=''):
    for name in sorted(zf.namelist()):
        if not name.startswith(subfolder):
            continue
-        #print name[n:]
+        # print name[n:]
        if name.endswith('/'):
            folder = os.path.join(dir, name[n:])
            if not os.path.exists(folder):
@@ -435,6 +438,7 @@ def add_path_first(path):
    if not global_settings.web2py_runtime_gae:
        site.addsitedir(path)

+
 def try_mkdir(path):
    if not os.path.exists(path):
        try:
@@ -444,11 +448,12 @@ def try_mkdir(path):
            else:
                os.mkdir(path)
        except OSError as e:
-            if e.strerror == 'File exists': # In case of race condition.
+            if e.strerror == 'File exists':  # In case of race condition.
                pass
            else:
                raise e

+
 def create_missing_folders():
    if not global_settings.web2py_runtime_gae:
        for path in ('applications', 'deposit', 'site-packages', 'logs'):
@@ -20,10 +20,10 @@ DEFAULT = lambda: None
 class AuthAPI(object):
    """
    AuthAPI is a barebones Auth implementation which does not have a concept of
-    HTML forms or redirects, emailing or even an URL, you are responsible for 
+    HTML forms or redirects, emailing or even an URL, you are responsible for
    all that if you use it.
    The main Auth functions such as login, logout, register, profile are designed
-    in a Dict In -> Dict Out logic so, for instance, if you set 
+    in a Dict In -> Dict Out logic so, for instance, if you set
    registration_requires_verification you are responsible for sending the key to
    the user and even rolling back the transaction if you can't do it.

@@ -115,7 +115,7 @@ class AuthAPI(object):
            if auth.last_visit and auth.last_visit + delta > now:
                self.user = auth.user
                # this is a trick to speed up sessions to avoid many writes
-                if (now - auth.last_visit).seconds > (auth.expiration / 10):
+                if (now - auth.last_visit).seconds > (auth.expiration // 10):
                    auth.last_visit = now
            else:
                self.user = None
@@ -245,13 +245,13 @@ class AuthAPI(object):
            migrate = db._migrate
        if fake_migrate is None:
            fake_migrate = db._fake_migrate
-        
+
        settings = self.settings
        if username is None:
            username = settings.use_username
        else:
            settings.use_username = username
-        
+
        if not self.signature:
            self.define_signature()
        if signature is True:
@@ -557,27 +557,43 @@ class AuthAPI(object):
        self.log_event(self.messages['del_membership_log'],
                       dict(user_id=user_id, group_id=group_id))
        ret = self.db(membership.user_id == user_id)(membership.group_id == group_id).delete()
-        if group_id in self.user_groups:
+        if group_id in self.user_groups and user_id == self.user_id:
            del self.user_groups[group_id]
        return ret

-    def has_membership(self, group_id=None, user_id=None, role=None):
+    def has_membership(self, group_id=None, user_id=None, role=None, cached=False):
        """
        Checks if user is member of group_id or role
+
+        NOTE: To avoid database query at each page load that use auth.has_membership, someone can use cached=True.
+              If cached is set to True has_membership() check group_id or role only against auth.user_groups variable
+              which is populated properly only at login time. This means that if an user membership change during a
+              given session the user has to log off and log in again in order to auth.user_groups to be properly
+              recreated and reflecting the user membership modification. There is one exception to this log off and
+              log in process which is in case that the user change his own membership, in this case auth.user_groups
+              can be properly update for the actual connected user because web2py has access to the proper session
+              user_groups variable. To make use of this exception someone has to place an "auth.update_groups()"
+              instruction in his app code to force auth.user_groups to be updated. As mention this will only work if it
+              the user itself that change it membership not if another user, let say an administrator, change someone
+              else's membership.
        """
-        group_id = group_id or self.id_group(role)
-        try:
-            group_id = int(group_id)
-        except:
-            group_id = self.id_group(group_id)  # interpret group_id as a role
        if not user_id and self.user:
            user_id = self.user.id
-        membership = self.table_membership()
-        if group_id and user_id and self.db((membership.user_id == user_id) &
-                                            (membership.group_id == group_id)).select():
-            r = True
+        if cached:
+            id_role = group_id or role
+            r = (user_id and id_role in self.user_groups.values()) or (user_id and id_role in self.user_groups)
        else:
-            r = False
+            group_id = group_id or self.id_group(role)
+            try:
+                group_id = int(group_id)
+            except:
+                group_id = self.id_group(group_id)  # interpret group_id as a role
+            membership = self.table_membership()
+            if group_id and user_id and self.db((membership.user_id == user_id) &
+                                                (membership.group_id == group_id)).select():
+                r = True
+            else:
+                r = False
        self.log_event(self.messages['has_membership_log'],
                       dict(user_id=user_id, group_id=group_id, check=r))
        return r
@@ -972,7 +988,8 @@ class AuthAPI(object):
            requires = [requires]
        requires = list(filter(lambda t: isinstance(t, CRYPT), requires))
        if requires:
-            requires[0].min_length = 0
+            requires[0] = CRYPT(**requires[0].__dict__) # Copy the existing CRYPT attributes
+            requires[0].min_length = 0 # But do not enforce minimum length for the old password

        old_password = kwargs.get('old_password', '')
        new_password = kwargs.get('new_password', '')
@@ -1012,7 +1029,7 @@ class AuthAPI(object):
                   ):
        """
        Verify a given registration_key actually exists in the user table.
-        Resets the key to empty string '' or 'pending' if 
+        Resets the key to empty string '' or 'pending' if
        setttings.registration_requires_approval is true.

        Keyword Args:
@@ -18,7 +18,7 @@ import fnmatch
 import os
 import copy
 import random
-from gluon._compat import builtin, PY2, unicodeT, to_native, to_bytes, iteritems, basestring, reduce, xrange, long
+from gluon._compat import builtin, PY2, unicodeT, to_native, to_bytes, iteritems, basestring, reduce, xrange, long, reload
 from gluon.storage import Storage, List
 from gluon.template import parse_template
 from gluon.restricted import restricted, compile2
@@ -41,11 +41,12 @@ import imp
 import logging
 import types
 from functools import reduce
-logger = logging.getLogger("web2py")
 from gluon import rewrite
 from gluon.custom_import import custom_import_install
 import py_compile

+logger = logging.getLogger("web2py")
+
 is_pypy = settings.global_settings.is_pypy
 is_gae = settings.global_settings.web2py_runtime_gae
 is_jython = settings.global_settings.is_jython
@@ -111,7 +112,7 @@ class mybuiltin(object):
    NOTE could simple use a dict and populate it,
    NOTE not sure if this changes things though if monkey patching import.....
    """
-    #__builtins__
+    # __builtins__
    def __getitem__(self, key):
        try:
            return getattr(builtin, key)
@@ -185,7 +186,7 @@ def LOAD(c=None, f='index', args=None, vars=None,
        else:
            statement = "$.web2py.component('%s','%s');" % (url, target)
        attr['_data-w2p_remote'] = url
-        if not target is None:
+        if target is not None:
            return DIV(content, **attr)

    else:
@@ -204,14 +205,15 @@ def LOAD(c=None, f='index', args=None, vars=None,
        other_response = Response()
        other_request.env.path_info = '/' + \
            '/'.join([request.application, c, f] +
-                     map(str, other_request.args))
+                     [str(a) for a in other_request.args])
        other_request.env.query_string = \
            vars and URL(vars=vars).split('?')[1] or ''
        other_request.env.http_web2py_component_location = \
            request.env.path_info
        other_request.cid = target
        other_request.env.http_web2py_component_element = target
-        other_request.restful = types.MethodType(request.restful.__func__, other_request) # A bit nasty but needed to use LOAD on action decorates with @request.restful()
+        other_request.restful = types.MethodType(request.restful.__func__, other_request)
+        # A bit nasty but needed to use LOAD on action decorates with @request.restful()
        other_response.view = '%s/%s.%s' % (c, f, other_request.extension)

        other_environment = copy.copy(current.globalenv)  # NASTY
@@ -286,7 +288,7 @@ class LoadFactory(object):
            other_response = globals.Response()
            other_request.env.path_info = '/' + \
                '/'.join([request.application, c, f] +
-                         map(str, other_request.args))
+                         [str(a) for a in other_request.args])
            other_request.env.query_string = \
                vars and html.URL(vars=vars).split('?')[1] or ''
            other_request.env.http_web2py_component_location = \
@@ -405,7 +407,7 @@ def build_environment(request, response, session, store_current=True):
    """
    Build the environment dictionary into which web2py files are executed.
    """
-    #h,v = html,validators
+    # h,v = html,validators
    environment = dict(_base_environment_)

    if not request.env:
@@ -418,7 +420,7 @@ def build_environment(request, response, session, store_current=True):
        r'^%s/%s/\w+\.py$' % (request.controller, request.function)
        ]

-    t = environment['T'] = translator(os.path.join(request.folder,'languages'),
+    t = environment['T'] = translator(os.path.join(request.folder, 'languages'),
                                      request.env.http_accept_language)
    c = environment['cache'] = Cache(request)

@@ -506,10 +508,12 @@ def compile_models(folder):
        save_pyc(filename)
        os.unlink(filename)

+
 def find_exposed_functions(data):
-    data = regex_longcomments.sub('',data)
+    data = regex_longcomments.sub('', data)
    return regex_expose.findall(data)

+
 def compile_controllers(folder):
    """
    Compiles all the controllers in the application specified by `folder`
@@ -524,16 +528,19 @@ def compile_controllers(folder):
            command = data + "\nresponse._vars=response._caller(%s)\n" % \
                function
            filename = pjoin(folder, 'compiled',
-                             'controllers.%s.%s.py' % (fname[:-3],function))
+                             'controllers.%s.%s.py' % (fname[:-3], function))
            write_file(filename, command)
            save_pyc(filename)
            os.unlink(filename)

+
 def model_cmp(a, b, sep='.'):
    return cmp(a.count(sep), b.count(sep)) or cmp(a, b)

+
 def model_cmp_sep(a, b, sep=os.path.sep):
-    return model_cmp(a,b,sep)
+    return model_cmp(a, b, sep)
+

 def run_models_in(environment):
    """
@@ -544,7 +551,7 @@ def run_models_in(environment):
    request = current.request
    folder = request.folder
    c = request.controller
-    #f = environment['request'].function
+    # f = environment['request'].function
    response = current.response

    path = pjoin(folder, 'models')
@@ -557,9 +564,11 @@ def run_models_in(environment):
            models = sorted(listdir(path, '^\w+\.py$', 0, sort=False), model_cmp_sep)
    else:
        if compiled:
-            models = sorted(listdir(cpath, '^models[_.][\w.]+\.pyc$', 0), key=lambda f: '{0:03d}'.format(f.count('.')) + f)
+            models = sorted(listdir(cpath, '^models[_.][\w.]+\.pyc$', 0),
+                            key=lambda f: '{0:03d}'.format(f.count('.')) + f)
        else:
-            models = sorted(listdir(path, '^\w+\.py$', 0, sort=False), key=lambda f: '{0:03d}'.format(f.count(os.path.sep)) + f)
+            models = sorted(listdir(path, '^\w+\.py$', 0, sort=False),
+                            key=lambda f: '{0:03d}'.format(f.count(os.path.sep)) + f)

    models_to_run = None
    for model in models:
@@ -570,10 +579,10 @@ def run_models_in(environment):
        if models_to_run:
            if compiled:
                n = len(cpath)+8
-                fname = model[n:-4].replace('.','/')+'.py'
+                fname = model[n:-4].replace('.', '/')+'.py'
            else:
                n = len(path)+1
-                fname = model[n:].replace(os.path.sep,'/')
+                fname = model[n:].replace(os.path.sep, '/')
            if not regex.search(fname) and c != 'appadmin':
                continue
            elif compiled:
@@ -583,6 +592,7 @@ def run_models_in(environment):
            ccode = getcfs(model, model, f)
            restricted(ccode, environment, layer=model)

+
 def run_controller_in(controller, function, environment):
    """
    Runs the controller.function() (for the app specified by
@@ -596,13 +606,13 @@ def run_controller_in(controller, function, environment):
    badc = 'invalid controller (%s/%s)' % (controller, function)
    badf = 'invalid function (%s/%s)' % (controller, function)
    if os.path.exists(cpath):
-        filename = pjoin(cpath, 'controllers.%s.%s.pyc'
-                         % (controller, function))
-        if not os.path.exists(filename):
+        filename = pjoin(cpath, 'controllers.%s.%s.pyc' % (controller, function))
+        try:
+            ccode = getcfs(filename, filename, lambda: read_pyc(filename))
+        except IOError:
            raise HTTP(404,
                       rewrite.THREAD_LOCAL.routes.error_message % badf,
                       web2py_error=badf)
-        ccode = getcfs(filename, filename, lambda: read_pyc(filename))
    elif function == '_TEST':
        # TESTING: adjust the path to include site packages
        from gluon.settings import global_settings
@@ -623,15 +633,15 @@ def run_controller_in(controller, function, environment):
        code += TEST_CODE
        ccode = compile2(code, filename)
    else:
-        filename = pjoin(folder, 'controllers/%s.py'
-                                 % controller)
-        if not os.path.exists(filename):
+        filename = pjoin(folder, 'controllers/%s.py' % controller)
+        try:
+            code = getcfs(filename, filename, lambda: read_file(filename))
+        except IOError:
            raise HTTP(404,
                       rewrite.THREAD_LOCAL.routes.error_message % badc,
                       web2py_error=badc)
-        code = getcfs(filename, filename, lambda: read_file(filename))
        exposed = find_exposed_functions(code)
-        if not function in exposed:
+        if function not in exposed:
            raise HTTP(404,
                       rewrite.THREAD_LOCAL.routes.error_message % badf,
                       web2py_error=badf)
@@ -666,8 +676,9 @@ def run_view_in(environment):
    badv = 'invalid view (%s)' % view
    patterns = response.get('generic_patterns')
    layer = None
+    scode = None
    if patterns:
-        regex = re_compile('|'.join(map(fnmatch.translate, patterns)))
+        regex = re_compile('|'.join(fnmatch.translate(p) for p in patterns))
        short_action = '%(controller)s/%(function)s.%(extension)s' % request
        allow_generic = regex.search(short_action)
    else:
@@ -678,7 +689,7 @@ def run_view_in(environment):
        layer = 'file stream'
    else:
        filename = pjoin(folder, 'views', view)
-        if os.path.exists(cpath): # compiled views
+        if os.path.exists(cpath):  # compiled views
            x = view.replace('/', '.')
            files = ['views.%s.pyc' % x]
            is_compiled = os.path.exists(pjoin(cpath, files[0]))
@@ -698,23 +709,27 @@ def run_view_in(environment):
                        ccode = getcfs(compiled, compiled, lambda: read_pyc(compiled))
                        layer = compiled
                        break
-        if not os.path.exists(filename) and allow_generic:
-            view = 'generic.' + request.extension
-            filename = pjoin(folder, 'views', view)
-        if not os.path.exists(filename):
-            raise HTTP(404,
-                       rewrite.THREAD_LOCAL.routes.error_message % badv,
-                       web2py_error=badv)
-        layer = filename
-        # Compile the template
-        ccode = parse_template(view,
-                               pjoin(folder, 'views'),
-                               context=environment)
-
-    restricted(ccode, environment, layer=layer)
+        # if the view is not compiled
+        if not layer:
+            if not os.path.exists(filename) and allow_generic:
+                view = 'generic.' + request.extension
+                filename = pjoin(folder, 'views', view)
+            if not os.path.exists(filename):
+                raise HTTP(404,
+                           rewrite.THREAD_LOCAL.routes.error_message % badv,
+                           web2py_error=badv)
+            # Parse template
+            scode = parse_template(view,
+                                   pjoin(folder, 'views'),
+                                   context=environment)
+            # Compile template
+            ccode = compile2(scode, filename)
+            layer = filename        
+    restricted(ccode, environment, layer=layer, scode=scode)
    # parse_template saves everything in response body
    return environment['response'].body.getvalue()

+
 def remove_compiled_application(folder):
    """
    Deletes the folder `compiled` containing the compiled application.
@@ -330,7 +330,7 @@ CONTENT_TYPE = {
    '.lha': 'application/x-lha',
    '.lhs': 'text/x-literate-haskell',
    '.lhz': 'application/x-lhz',
-    '.load' : 'text/html',
+    '.load': 'text/html',
    '.log': 'text/x-log',
    '.lrz': 'application/x-lrzip',
    '.ltx': 'text/x-tex',
@@ -823,7 +823,7 @@ CONTENT_TYPE = {
    '.xsd': 'application/xml',
    '.xsl': 'application/xslt+xml',
    '.xslfo': 'text/x-xslfo',
-    '.xslm' : 'application/vnd.ms-excel.sheet.macroEnabled.12',
+    '.xslm': 'application/vnd.ms-excel.sheet.macroEnabled.12',
    '.xslt': 'application/xslt+xml',
    '.xspf': 'application/xspf+xml',
    '.xul': 'application/vnd.mozilla.xul+xml',
@@ -843,7 +843,7 @@ def contenttype(filename, default='text/plain'):
    """
    Returns the Content-Type string matching extension of the given filename.
    """
-    filename=to_native(filename)
+    filename = to_native(filename)
    i = filename.rfind('.')
    if i >= 0:
        default = CONTENT_TYPE.get(filename[i:].lower(), default)
@@ -7,15 +7,13 @@ db = get_db()
 """
 import os
 from gluon import *
-from pydal.adapters import ADAPTERS, PostgreSQLAdapter
-from pydal.helpers.classes import UseDatabaseStoredFile
+from pydal.adapters import adapters, PostgrePsyco
+from pydal.helpers.classes import DatabaseStoredFile

-class HerokuPostgresAdapter(UseDatabaseStoredFile,PostgreSQLAdapter):
-    drivers = ('psycopg2',)
+@adapters.register_for('postgres')
+class HerokuPostgresAdapter(DatabaseStoredFile, PostgrePsyco):
    uploads_in_blob = True

-ADAPTERS['postgres'] = HerokuPostgresAdapter
-
 def get_db(name = None, pool_size=10):
    if not name:
        names = [n for n in os.environ.keys()
@@ -45,7 +45,7 @@ class RESIZE(object):
                background = Image.new('RGBA', (self.nx, self.ny), (255, 255, 255, 0))
                background.paste(
                    img,
-                    ((self.nx - img.size[0]) / 2, (self.ny - img.size[1]) / 2))
+                    ((self.nx - img.size[0]) // 2, (self.ny - img.size[1]) // 2))
                background.save(s, 'JPEG', quality=self.quality)
            else:
                img.save(s, 'JPEG', quality=self.quality)
@@ -49,7 +49,8 @@ class CasAuth(object):
                           email=lambda v: v.get('email', None),
                           user_id=lambda v: v['user']),
                 casversion=1,
-                 casusername='cas:user'
+                 casusername='cas:user',
+		 change_password_url=None
                 ):
        self.urlbase = urlbase
        self.cas_login_url = "%s/%s" % (self.urlbase, actions[0])
@@ -64,6 +65,9 @@ class CasAuth(object):
                              #vars=current.request.vars,
                              scheme=True)

+	# URL to let users change their password in the IDP system
+        self.cas_change_password_url = change_password_url
+
    def login_url(self, next="/"):
        current.session.token = self._CAS_login()
        return next
@@ -74,6 +78,10 @@ class CasAuth(object):
        self._CAS_logout()
        return next

+    def change_password_url(self, next="/"):
+        self._CAS_change_password()
+        return next
+
    def get_user(self):
        user = current.session.token
        if user:
@@ -135,3 +143,6 @@ class CasAuth(object):
        redirects to the CAS logout page
        """
        redirect("%s?service=%s" % (self.cas_logout_url, self.cas_my_url))
+
+    def _CAS_change_password(self):
+        redirect(self.cas_change_password_url)
@@ -145,10 +145,16 @@ class Saml2Auth(object):
            username=lambda v:v['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/upn'][0],
            email=lambda v:v['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/upn'][0],
            user_id=lambda v:v['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/upn'][0],
-            )):
+            ), logout_url=None, change_password_url=None):
        self.config_file = config_file
        self.maps = maps

+	# URL for redirecting users to when they sign out
+        self.saml_logout_url = logout_url
+
+        # URL to let users change their password in the IDP system
+        self.saml_change_password_url = change_password_url
+
    def login_url(self, next="/"):
        d = saml2_handler(current.session, current.request)
        if 'url' in d:
@@ -170,6 +176,12 @@ class Saml2Auth(object):

    def logout_url(self, next="/"):
        current.session.saml2_info = None
+        current.session.auth = None
+        self._SAML_logout()
+        return next
+
+    def change_password_url(self, next="/"):
+        self._SAML_change_password()
        return next

    def get_user(self):        
@@ -180,3 +192,13 @@ class Saml2Auth(object):
                d[key] = self.maps[key](user)
            return d
        return None
+
+    def _SAML_logout(self):
+        """
+        exposed SAML.logout()
+        redirects to the SAML logout page
+        """
+        redirect(self.saml_logout_url)
+
+    def _SAML_change_password(self):
+        redirect(self.saml_change_password_url)
@@ -5,12 +5,19 @@
 # license MIT/BSD/GPL
 from __future__ import print_function
 import re
+import sys
 import urllib
-from gluon._compat import maketrans, urllib_quote, unicodeT, to_bytes, to_native, xrange
-from gluon.utils import local_html_escape as escape
-from ast import parse as ast_parse
 import ast

+PY2 = sys.version_info[0] == 2
+
+if PY2:
+    from urllib import quote as urllib_quote
+    from string import maketrans
+else:
+    from urllib.parse import quote as urllib_quote
+    maketrans = str.maketrans
+

 """
 TODO: next version should use MathJax
@@ -564,6 +571,29 @@ ttab_in = maketrans("'`:*~\\[]{}@$+-.#\n", '\x0b\x0c\x0e\x0f\x10\x11\x12\x13\x14
 ttab_out = maketrans('\x0b\x0c\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x05', "'`:*~\\[]{}@$+-.#\n")
 regex_quote = re.compile('(?P<name>\w+?)\s*\=\s*')

+def local_html_escape(data, quote=False):
+    """
+    Works with bytes.
+    Replace special characters "&", "<" and ">" to HTML-safe sequences.
+    If the optional flag quote is true (the default), the quotation mark
+    characters, both double quote (") and single quote (') characters are also
+    translated.
+    """
+    if PY2:
+        import cgi
+        data = cgi.escape(data, quote)
+        return data.replace("'", "&#x27;") if quote else data
+    else:
+        import html
+        if isinstance(data, str):
+            return html.escape(data, quote=quote)
+        data = data.replace(b"&", b"&amp;")  # Must be done first!                                                                                           
+        data = data.replace(b"<", b"&lt;")
+        data = data.replace(b">", b"&gt;")
+        if quote:
+            data = data.replace(b'"', b"&quot;")
+            data = data.replace(b'\'', b"&#x27;")
+        return data

 def make_dict(b):
    return '{%s}' % regex_quote.sub("'\g<name>':", b)
@@ -579,7 +609,7 @@ def safe_eval(node_or_string, env):
    _safe_names = {'None': None, 'True': True, 'False': False}
    _safe_names.update(env)
    if isinstance(node_or_string, basestring):
-        node_or_string = ast_parse(node_or_string, mode='eval')
+        node_or_string = ast.parse(node_or_string, mode='eval')
    if isinstance(node_or_string, ast.Expression):
        node_or_string = node_or_string.body

@@ -599,14 +629,14 @@ def safe_eval(node_or_string, env):
            if node.id in _safe_names:
                return _safe_names[node.id]
        elif isinstance(node, ast.BinOp) and \
-                isinstance(node.op, (Add, Sub)) and \
-                isinstance(node.right, Num) and \
+                isinstance(node.op, (ast.Add, ast.Sub)) and \
+                isinstance(node.right, ast.Num) and \
                isinstance(node.right.n, complex) and \
-                isinstance(node.left, Num) and \
+                isinstance(node.left, ast.Num) and \
                isinstance(node.left.n, (int, long, float)):
            left = node.left.n
            right = node.right.n
-            if isinstance(node.op, Add):
+            if isinstance(node.op, ast.Add):
                return left + right
            else:
                return left - right
@@ -765,7 +795,7 @@ def render(text,
    '<table><tbody><tr class="first"><td>a</td><td>b</td></tr><tr class="even"><td>c</td><td>d</td></tr></tbody></table>'

    >>> render("----\\nhello world\\n----\\n")
-    '<blockquote>hello world</blockquote>'
+    '<blockquote><p>hello world</p></blockquote>'

    >>> render('[[myanchor]]')
    '<p><span class="anchor" id="markmin_myanchor"></span></p>'
@@ -946,7 +976,8 @@ def render(text,
    if protolinks == "default":
        protolinks = protolinks_simple
    pp = '\n' if pretty_print else ''
-    text = to_native(text)
+    text = text if text is None or isinstance(text, str) else text.decode('utf8', 'strict')
+
    if not (isinstance(text, str)):
        text = str(text or '')
    text = regex_backslash.sub(lambda m: m.group(1).translate(ttab_in), text)
@@ -994,7 +1025,7 @@ def render(text,
        return LINK

    text = regex_link.sub(mark_link, text)
-    text = escape(text)
+    text = local_html_escape(text)

    if protolinks:
        text = regex_proto.sub(lambda m: protolinks(*m.group('p', 'k')), text)
@@ -1035,7 +1066,7 @@ def render(text,
            if pend and mtag == '.':  # paragraph in a list:
                out.append(etags.pop())
                ltags.pop()
-            for i in xrange(lent - lev):
+            for i in range(lent - lev):
                out.append('<' + tag + '>' + pp)
                etags.append('</' + tag + '>' + pp)
                lev += 1
@@ -1044,7 +1075,7 @@ def render(text,
        elif lent == lev:
            if tlev[-1] != tag:
                # type of list is changed (ul<=>ol):
-                for i in xrange(ltags.count(lent)):
+                for i in range(ltags.count(lent)):
                    ltags.pop()
                    out.append(etags.pop())
                tlev[-1] = tag
@@ -1209,7 +1240,7 @@ def render(text,
                s = '<blockquote%s%s>%s</blockquote>%s' \
                    % (t_cls,
                       t_id,
-                       '\n'.join(strings[bq_begin:lineno]), pp)
+                       render('\n'.join(strings[bq_begin:lineno])), pp)
                mtag = 'q'
        else:
            s = '<hr />'
@@ -1322,10 +1353,10 @@ def render(text,
        t, a, k, p, w = m.group('t', 'a', 'k', 'p', 'w')
        if not k:
            return m.group(0)
-        k = escape(k)
+        k = local_html_escape(k)
        t = t or ''
        style = 'width:%s' % w if w else ''
-        title = ' title="%s"' % escape(a).replace(META, DISABLED_META) if a else ''
+        title = ' title="%s"' % local_html_escape(a).replace(META, DISABLED_META) if a else ''
        p_begin = p_end = ''
        if p == 'center':
            p_begin = '<p style="text-align:center">'
@@ -1349,7 +1380,7 @@ def render(text,
                       autolinks, protolinks, class_prefix, id_prefix, pretty_print)
            return '<%(p)s controls="controls"%(title)s%(style)s><source src="%(k)s" />%(t)s</%(p)s>' \
                   % dict(p=p, title=title, style=style, k=k, t=t)
-        alt = ' alt="%s"' % escape(t).replace(META, DISABLED_META) if t else ''
+        alt = ' alt="%s"' % local_html_escape(t).replace(META, DISABLED_META) if t else ''
        return '%(begin)s<img src="%(k)s"%(alt)s%(title)s%(style)s />%(end)s' \
               % dict(begin=p_begin, k=k, alt=alt, title=title, style=style, end=p_end)

@@ -1358,12 +1389,12 @@ def render(text,
        if not k and not t:
            return m.group(0)
        t = t or ''
-        a = escape(a) if a else ''
+        a = local_html_escape(a) if a else ''
        if k:
            if '#' in k and ':' not in k.split('#')[0]:
                # wikipage, not external url
                k = k.replace('#', '#' + id_prefix)
-            k = escape(k)
+            k = local_html_escape(k)
            title = ' title="%s"' % a.replace(META, DISABLED_META) if a else ''
            target = ' target="_blank"' if p == 'popup' else ''
            t = render(t, {}, {}, 'br', URL, environment, latex, None,
@@ -1373,7 +1404,7 @@ def render(text,
        if t == 'NEWLINE' and not a:
            return '<br />' + pp
        return '<span class="anchor" id="%s">%s</span>' % (
-            escape(id_prefix + t),
+            local_html_escape(id_prefix + t),
            render(a, {}, {}, 'br', URL,
                   environment, latex, autolinks,
                   protolinks, class_prefix,
@@ -1399,7 +1430,7 @@ def render(text,
    def expand_meta(m):
        code, b, p, s = segments.pop(0)
        if code is None or m.group() == DISABLED_META:
-            return escape(s)
+            return local_html_escape(s)
        if b in extra:
            if code[:1] == '\n':
                code = code[1:]
@@ -1411,7 +1442,7 @@ def render(text,
                return str(extra[b](code))
        elif b == 'cite':
            return '[' + ','.join('<a href="#%s" class="%s">%s</a>' %
-                                  (id_prefix + d, b, d) for d in escape(code).split(',')) + ']'
+                                  (id_prefix + d, b, d) for d in local_html_escape(code).split(',')) + ']'
        elif b == 'latex':
            return LATEX % urllib_quote(code)
        elif b in html_colors:
@@ -1426,12 +1457,12 @@ def render(text,
                   % (fg, bg, render(code, {}, {}, 'br', URL, environment, latex,
                                     autolinks, protolinks, class_prefix, id_prefix, pretty_print))
        cls = ' class="%s%s"' % (class_prefix, b) if b and b != 'id' else ''
-        id = ' id="%s%s"' % (id_prefix, escape(p)) if p else ''
+        id = ' id="%s%s"' % (id_prefix, local_html_escape(p)) if p else ''
        beg = (code[:1] == '\n')
        end = [None, -1][code[-1:] == '\n']
        if beg and end:
-            return '<pre><code%s%s>%s</code></pre>%s' % (cls, id, escape(code[1:-1]), pp)
-        return '<code%s%s>%s</code>' % (cls, id, escape(code[beg:end]))
+            return '<pre><code%s%s>%s</code></pre>%s' % (cls, id, local_html_escape(code[1:-1]), pp)
+        return '<code%s%s>%s</code>' % (cls, id, local_html_escape(code[beg:end]))

    text = regex_expand_meta.sub(expand_meta, text)

@@ -10,8 +10,11 @@ Original author: Zachary Voase
 Modified for inclusion into web2py by: Ross Peoples <ross.peoples@gmail.com>
 """

+try:
+    from StringIO import StringIO
+except ImportError:
+    from io import StringIO

-from StringIO import StringIO  # The pure-Python StringIO supports unicode.
 import re


@@ -8,30 +8,40 @@ Created by: Ross Peoples <ross.peoples@gmail.com>
 Modified by: Massimo Di Pierro <massimo.dipierro@gmail.com>
 """

-import cssmin
-import jsmin
+from . import cssmin
+from . import jsmin
 import os
 import hashlib
 import re
+import sys
+PY2 = sys.version_info[0] == 2

+if PY2:
+    hashlib_md5 = hashlib.md5
+else:
+    hashlib_md5 = lambda s: hashlib.md5(bytes(s, 'utf8'))
+
+def open_py23(filename, mode):
+    if PY2:
+        f = open(filename, mode + 'b')
+    else:
+        f = open(filename, mode, encoding="utf8")
+    return f

 def read_binary_file(filename):
-    f = open(filename, 'rb')
+    f = open_py23(filename, 'r')
    data = f.read()
    f.close()
    return data

-
 def write_binary_file(filename, data):
-    f = open(filename, 'wb')
+    f = open_py23(filename, 'w')
    f.write(data)
    f.close()

-
 def fix_links(css, static_path):
    return re.sub(r'url\((["\'])\.\./', 'url(\\1' + static_path, css)

-
 def minify(files, path_info, folder, optimize_css, optimize_js,
           ignore_concat=[],
           ignore_minify=['/jquery.js', '/anytime.js']):
@@ -109,7 +119,7 @@ def minify(files, path_info, folder, optimize_css, optimize_js,
                    js.append(contents)
            else:
                js.append(filename)
-    dest_key = hashlib.md5(repr(processed)).hexdigest()
+    dest_key = hashlib_md5(repr(processed)).hexdigest()
    if css and concat_css:
        css = '\n\n'.join(contents for contents in css)
        if not inline_css:
@@ -137,6 +137,8 @@ def populate_generator(table, default=True, compute=False, contents={}):
                continue
            elif field.type == 'upload':
                continue
+            elif field.compute is not None:
+                continue
            elif default and not field.default in (None, ''):
                record[fieldname] = field.default
            elif compute and field.compute:
@@ -673,3 +673,23 @@ def simple_detect(agent):
    if os_version:
        os = " ".join((os, os_version))
    return os, browser
+
+
+class mobilize(object):  
+    """
+    Decorator for controller functions so they use different views for mobile devices.
+
+    WARNING: If you update httpagentparser make sure to leave mobilize for
+    backwards compatibility.
+    """
+    def __init__(self, func):
+        self.func = func
+    
+    def __call__(self):
+        from gluon import current
+        user_agent = current.request.user_agent()
+        if user_agent.is_mobile:
+            items = current.response.view.split('.')
+            items.insert(-1, 'mobile')
+            current.response.view = '.'.join(items)
+        return self.func()
@@ -94,32 +94,10 @@ import optparse
 import time
 import sys
 import gluon.utils
-
-if (sys.version_info[0] == 2):
-    from urllib import urlencode, urlopen
-    def to_bytes(obj, charset='utf-8', errors='strict'):
-        if obj is None:
-            return None
-        if isinstance(obj, (bytes, bytearray, buffer)):
-            return bytes(obj)
-        if isinstance(obj, unicode):
-            return obj.encode(charset, errors)
-        raise TypeError('Expected bytes')
-else:
-    from urllib.request import urlopen
-    from urllib.parse import urlencode
-    def to_bytes(obj, charset='utf-8', errors='strict'):
-        if obj is None:
-            return None
-        if isinstance(obj, (bytes, bytearray, memoryview)):
-            return bytes(obj)
-        if isinstance(obj, str):
-            return obj.encode(charset, errors)
-        raise TypeError('Expected bytes')
+from gluon._compat import to_native, to_bytes, urlencode, urlopen

 listeners, names, tokens = {}, {}, {}

-
 def websocket_send(url, message, hmac_key=None, group='default'):
    sig = hmac_key and hmac.new(to_bytes(hmac_key), to_bytes(message)).hexdigest() or ''
    params = urlencode(
@@ -138,8 +116,8 @@ class PostHandler(tornado.web.RequestHandler):
        if hmac_key and not 'signature' in self.request.arguments:
            self.send_error(401)
        if 'message' in self.request.arguments:
-            message = self.request.arguments['message'][0]
-            group = self.request.arguments.get('group', ['default'])[0]
+            message = self.request.arguments['message'][0].decode(encoding='UTF-8')
+            group = self.request.arguments.get('group', ['default'])[0].decode(encoding='UTF-8')
            print('%s:MESSAGE to %s:%s' % (time.time(), group, message))
            if hmac_key:
                signature = self.request.arguments['signature'][0]
@@ -8,7 +8,7 @@
 Support for smart import syntax for web2py applications
 -------------------------------------------------------
 """
-from gluon._compat import builtin, unicodeT, PY2, to_native
+from gluon._compat import builtin, unicodeT, PY2, to_native, reload
 import os
 import sys
 import threading
@@ -14,6 +14,12 @@ from pydal import DAL as DAL
 from pydal import Field
 from pydal.objects import Row, Rows, Table, Query, Set, Expression
 from pydal import SQLCustomType, geoPoint, geoLine, geoPolygon
+from pydal.migrator import Migrator, InDBMigrator
+from gluon.serializers import custom_json, xml
+from gluon.utils import web2py_uuid
+from gluon import sqlhtml
+from pydal.drivers import DRIVERS
+

 def _default_validators(db, field):
    """
@@ -26,7 +32,11 @@ def _default_validators(db, field):
    field_type, field_length = field.type, field.length
    requires = []

-    if field_type in (('string', 'text', 'password')):
+    if isinstance(field.options, list) and field.requires:
+        requires = validators.IS_IN_SET(field.options, multiple=field_type.startswith('list:'))
+    elif field.regex and not field.requires:
+        requires.append(validators.IS_REGEX(regex))
+    elif field_type in (('string', 'text', 'password')):
        requires.append(validators.IS_LENGTH(field_length))
    elif field_type == 'json':
        requires.append(validators.IS_EMPTY_OR(validators.IS_JSON()))
@@ -44,48 +54,64 @@ def _default_validators(db, field):
        requires.append(validators.IS_TIME())
    elif field_type == 'datetime':
        requires.append(validators.IS_DATETIME())
-    elif db and field_type.startswith('reference') and \
-            field_type.find('.') < 0 and \
-            field_type[10:] in db.tables:
-        referenced = db[field_type[10:]]
-        if hasattr(referenced, '_format') and referenced._format:
-            requires = validators.IS_IN_DB(db, referenced._id,
-                                           referenced._format)
-            if field.unique:
-                requires._and = validators.IS_NOT_IN_DB(db, field)
-            if field.tablename == field_type[10:]:
-                return validators.IS_EMPTY_OR(requires)
-            return requires
-    elif db and field_type.startswith('list:reference') and \
-            field_type.find('.') < 0 and \
-            field_type[15:] in db.tables:
-        referenced = db[field_type[15:]]
-        if hasattr(referenced, '_format') and referenced._format:
-            requires = validators.IS_IN_DB(db, referenced._id,
-                                           referenced._format, multiple=True)
-        else:
-            requires = validators.IS_IN_DB(db, referenced._id,
-                                           multiple=True)
+    elif db and field_type.startswith('reference'):
+        if field_type.find('.') < 0 and field_type[10:] in db.tables:
+            referenced = db[field_type[10:]]
+            if hasattr(referenced, '_format') and referenced._format:
+                requires = validators.IS_IN_DB(db, referenced._id,referenced._format)
+            else:
+                requires = validators.IS_IN_DB(db, referenced._id)
+        elif field_type.find('.') > 0 and field_type[10:].split('.')[0] in db.tables:
+            table_field = field_type[10:].split('.')
+            table_name=table_field[0]
+            field_name=table_field[1]
+            referenced = db[table_name]
+            if hasattr(referenced, '_format') and referenced._format:
+                requires = validators.IS_IN_DB(db, referenced[field_name],referenced._format)
+            else:
+                requires = validators.IS_IN_DB(db, referenced[field_name])
        if field.unique:
            requires._and = validators.IS_NOT_IN_DB(db, field)
        if not field.notnull:
            requires = validators.IS_EMPTY_OR(requires)
        return requires
+    elif db and field_type.startswith('list:reference'):
+        if field_type.find('.') < 0 and field_type[15:] in db.tables:
+            referenced = db[field_type[15:]]
+            if hasattr(referenced, '_format') and referenced._format:
+                requires = validators.IS_IN_DB(db, referenced._id,
+                                               referenced._format, multiple=True)
+            else:
+                requires = validators.IS_IN_DB(db, referenced._id,
+                                               multiple=True)
+        elif field_type.find('.') > 0 and field_type[15:].split('.')[0] in db.tables:
+            table_field = field_type[15:].split('.')
+            table_name=table_field[0]
+            field_name=table_field[1]
+            referenced = db[table_name]
+            if hasattr(referenced, '_format') and referenced._format:
+                requires = validators.IS_IN_DB(db, referenced[field_name],
+                                               referenced._format, multiple=True)
+            else:
+                requires = validators.IS_IN_DB(db, referenced[field_name],
+                                               multiple=True)
+        if field.unique:
+            requires._and = validators.IS_NOT_IN_DB(db, field)
+        if not field.notnull:
+            requires = validators.IS_EMPTY_OR(requires)
+        return requires    
    # does not get here for reference and list:reference
-    if field.unique:
-        requires.insert(0, validators.IS_NOT_IN_DB(db, field))
-    excluded_fields = ['string', 'upload', 'text', 'password', 'boolean']
-    if (field.notnull or field.unique) and field_type not in excluded_fields:
-        requires.insert(0, validators.IS_NOT_EMPTY())
-    elif not field.notnull and not field.unique and requires:
-        requires[0] = validators.IS_EMPTY_OR(requires[0], null='' if field.type in ('string', 'text', 'password') else None)
+    if isinstance(requires, list):
+        if field.unique:
+            requires.insert(0, validators.IS_NOT_IN_DB(db, field))
+        excluded_fields = ['string', 'upload', 'text', 'password', 'boolean']
+        if (field.notnull or field.unique) and field_type not in excluded_fields:
+            requires.insert(0, validators.IS_NOT_EMPTY())
+        elif not field.notnull and not field.unique and requires:
+            null = null='' if field.type in ('string', 'text', 'password') else None
+            requires[0] = validators.IS_EMPTY_OR(requires[0], null=null)
    return requires

-from gluon.serializers import custom_json, xml
-from gluon.utils import web2py_uuid
-from gluon import sqlhtml
-
-
 DAL.serializers = {'json': custom_json, 'xml': xml}
 DAL.validators_method = _default_validators
 DAL.uuid = lambda x: web2py_uuid()
@@ -96,8 +122,7 @@ DAL.representers = {
 DAL.Field = Field
 DAL.Table = Table

-#: add web2py contrib drivers to pyDAL
-from pydal.drivers import DRIVERS
+# add web2py contrib drivers to pyDAL
 if not DRIVERS.get('pymysql'):
    try:
        from .contrib import pymysql
@@ -15,13 +15,13 @@ import codecs
 # None represents a potentially variable byte. "##" in the XML spec...
 autodetect_dict = {  # bytepattern     : ("name",
                                         (0x00, 0x00, 0xFE, 0xFF): ("ucs4_be"),
-    (0xFF, 0xFE, 0x00, 0x00): ("ucs4_le"),
-    (0xFE, 0xFF, None, None): ("utf_16_be"),
-    (0xFF, 0xFE, None, None): ("utf_16_le"),
-    (0x00, 0x3C, 0x00, 0x3F): ("utf_16_be"),
-    (0x3C, 0x00, 0x3F, 0x00): ("utf_16_le"),
-    (0x3C, 0x3F, 0x78, 0x6D): ("utf_8"),
-    (0x4C, 0x6F, 0xA7, 0x94): ("EBCDIC")
+                                         (0xFF, 0xFE, 0x00, 0x00): ("ucs4_le"),
+                                         (0xFE, 0xFF, None, None): ("utf_16_be"),
+                                         (0xFF, 0xFE, None, None): ("utf_16_le"),
+                                         (0x00, 0x3C, 0x00, 0x3F): ("utf_16_be"),
+                                         (0x3C, 0x00, 0x3F, 0x00): ("utf_16_le"),
+                                         (0x3C, 0x3F, 0x78, 0x6D): ("utf_8"),
+                                         (0x4C, 0x6F, 0xA7, 0x94): ("EBCDIC")
 }


@@ -36,10 +36,10 @@ def autoDetectXMLEncoding(buffer):
    # buffer at once but otherwise we'd have to decode a character at
    # a time looking for the quote character...that's a pain

-    encoding = "utf_8"    # according to the XML spec, this is the default
-                          # this code successively tries to refine the default
-                          # whenever it fails to refine, it falls back to
-                          # the last place encoding was set.
+    encoding = "utf_8"
+    # according to the XML spec, this is the default this code successively tries to refine the default
+    # whenever it fails to refine, it falls back to the last place encoding was set.
+
    if len(buffer) >= 4:
        bytes = (byte1, byte2, byte3, byte4) = tuple(map(ord, buffer[0:4]))
        enc_info = autodetect_dict.get(bytes, None)
@@ -51,8 +51,7 @@ def autoDetectXMLEncoding(buffer):
        enc_info = None

    if enc_info:
-        encoding = enc_info  # we've got a guess... these are
-                             #the new defaults
+        encoding = enc_info  # we've got a guess... these are the new defaults

        # try to find a more precise encoding using xml declaration
        secret_decoder_ring = codecs.lookup(encoding)[1]
@@ -415,10 +415,10 @@ def fix_newlines(path):
 |\r|
 )''')
    for filename in listdir(path, '.*\.(py|html)$', drop=False):
-        rdata = read_file(filename, 'rb')
+        rdata = read_file(filename, 'r')
        wdata = regex.sub('\n', rdata)
        if wdata != rdata:
-            write_file(filename, wdata, 'wb')
+            write_file(filename, wdata, 'w')


 def copystream(
@@ -13,7 +13,8 @@ Contains the classes for the global used variables:
 - Session

 """
-from gluon._compat import pickle, StringIO, copyreg, Cookie, urlparse, PY2, iteritems, to_unicode, to_native, unicodeT, long
+from gluon._compat import pickle, StringIO, copyreg, Cookie, urlparse, PY2, iteritems, to_unicode, to_native, \
+    unicodeT, long, hashlib_md5, urllib_quote
 from gluon.storage import Storage, List
 from gluon.streamer import streamer, stream_file_or_304_or_206, DEFAULT_CHUNK_SIZE
 from gluon.contenttype import contenttype
@@ -30,7 +31,7 @@ from gluon.fileutils import copystream
 import hashlib
 from pydal.contrib import portalocker
 from pickle import Pickler, MARK, DICT, EMPTY_DICT
-#from types import DictionaryType
+# from types import DictionaryType
 import datetime
 import re
 import os
@@ -48,7 +49,7 @@ PAST = 'Sat, 1-Jan-1971 00:00:00'
 FUTURE = 'Tue, 1-Dec-2999 23:59:59'

 try:
-    #FIXME PY3
+    # FIXME PY3
    from gluon.contrib.minify import minify
    have_minify = True
 except ImportError:
@@ -79,6 +80,7 @@ template_mapping = {
    'js:inline': js_inline
 }

+
 # IMPORTANT:
 # this is required so that pickled dict(s) and class.__dict__
 # are sorted and web2py can detect without ambiguity when a session changes
@@ -95,6 +97,7 @@ else:
    SortingPickler.dispatch_table = copyreg.dispatch_table.copy()
    SortingPickler.dispatch_table[dict] = SortingPickler.save_dict

+
 def sorting_dumps(obj, protocol=None):
    file = StringIO()
    SortingPickler(file, protocol).dump(obj)
@@ -120,7 +123,7 @@ def copystream_progress(request, chunk_size=10 ** 5):
        dest = tempfile.NamedTemporaryFile()
    except NotImplementedError:  # and GAE this
        dest = tempfile.TemporaryFile()
-    if not 'X-Progress-ID' in request.get_vars:
+    if 'X-Progress-ID' not in request.get_vars:
        copystream(source, dest, size, chunk_size)
        return dest
    cache_key = 'X-Progress-ID:' + request.get_vars['X-Progress-ID']
@@ -198,7 +201,8 @@ class Request(Storage):
        """Takes the QUERY_STRING and unpacks it to get_vars
        """
        query_string = self.env.get('query_string', '')
-        dget = urlparse.parse_qs(query_string, keep_blank_values=1)  # Ref: https://docs.python.org/2/library/cgi.html#cgi.parse_qs
+        dget = urlparse.parse_qs(query_string, keep_blank_values=1)
+        # Ref: https://docs.python.org/2/library/cgi.html#cgi.parse_qs
        get_vars = self._get_vars = Storage(dget)
        for (key, value) in iteritems(get_vars):
            if isinstance(value, list) and len(value) == 1:
@@ -216,7 +220,12 @@ class Request(Storage):

        if is_json:
            try:
-                json_vars = json_parser.load(body)
+                # In Python 3 versions prior to 3.6 load doesn't accept bytes and
+                # bytearray, so we read the body convert to native and use loads
+                # instead of load.
+                # This line can be simplified to json_vars = json_parser.load(body)
+                # if and when we drop support for python versions under 3.6
+                json_vars = json_parser.loads(to_native(body.read())) 
            except:
                # incoherent request bodies can still be parsed "ad-hoc"
                json_vars = {}
@@ -228,8 +237,7 @@ class Request(Storage):
            body.seek(0)

        # parse POST variables on POST, PUT, BOTH only in post_vars
-        if (body and not is_json
-            and env.request_method in ('POST', 'PUT', 'DELETE', 'BOTH')):
+        if body and not is_json and env.request_method in ('POST', 'PUT', 'DELETE', 'BOTH'):
            query_string = env.pop('QUERY_STRING', None)
            dpost = cgi.FieldStorage(fp=body, environ=env, keep_blank_values=1)
            try:
@@ -328,11 +336,16 @@ class Request(Storage):
        user_agent = session._user_agent
        if user_agent:
            return user_agent
-        user_agent = user_agent_parser.detect(self.env.http_user_agent)
+        http_user_agent = self.env.http_user_agent or ''
+        user_agent = user_agent_parser.detect(http_user_agent)
        for key, value in user_agent.items():
            if isinstance(value, dict):
                user_agent[key] = Storage(value)
-        user_agent = session._user_agent = Storage(user_agent)
+        user_agent = Storage(user_agent)
+        user_agent.is_mobile = 'Mobile' in http_user_agent
+        user_agent.is_tablet = 'Tablet' in http_user_agent
+        session._user_agent = user_agent 
+        
        return user_agent

    def requires_https(self):
@@ -355,7 +368,7 @@ class Request(Storage):
            def f(_action=action, *a, **b):
                request.is_restful = True
                env = request.env
-                is_json = env.content_type=='application/json'
+                is_json = env.content_type == 'application/json'
                method = env.request_method
                if not ignore_extension and len(request.args) and '.' in request.args[-1]:
                    request.args[-1], _, request.extension = request.args[-1].rpartition('.')
@@ -451,13 +464,13 @@ class Response(Storage):
        for meta in iteritems((self.meta or {})):
            k, v = meta
            if isinstance(v, dict):
-                s += '<meta' + ''.join(' %s="%s"' % (xmlescape(key), to_native(xmlescape(v[key]))) for key in v) +' />\n'
+                s += '<meta' + ''.join(' %s="%s"' % (xmlescape(key),
+                                                     to_native(xmlescape(v[key]))) for key in v) + ' />\n'
            else:
                s += '<meta name="%s" content="%s" />\n' % (k, to_native(xmlescape(v)))
        self.write(s, escape=False)

    def include_files(self, extensions=None):
-
        """
        Includes files (usually in the head).
        Can minify and cache local files
@@ -465,46 +478,67 @@ class Response(Storage):
        response.cache_includes = (cache_method, time_expire).
        Example: (cache.disk, 60) # caches to disk for 1 minute.
        """
+        app = current.request.application
+
+        # We start by building a files list in which adjacent files internal to
+        # the application are placed in a list inside the files list.
+        #
+        # We will only minify and concat adjacent internal files as there's
+        # no way to know if changing the order with which the files are apppended
+        # will break things since the order matters in both CSS and JS and 
+        # internal files may be interleaved with external ones.
        files = []
-        ext_files = []
-        has_js = has_css = False
+        # For the adjacent list we're going to use storage List to both distinguish
+        # from the regular list and so we can add attributes
+        internal = List()  
+        internal.has_js = False
+        internal.has_css = False
+        done = set() # to remove duplicates
        for item in self.files:
-            if isinstance(item, (list, tuple)):
-                ext_files.append(item)
+            if not isinstance(item, list):
+                if item in done:
+                    continue
+                done.add(item)
+            if isinstance(item, (list, tuple)) or not item.startswith('/' + app): # also consider items in other web2py applications to be external
+                if internal:
+                    files.append(internal)
+                    internal = List()
+                    internal.has_js = False
+                    internal.has_css = False
+                files.append(item)
                continue
            if extensions and not item.rpartition('.')[2] in extensions:
                continue
-            if item in files:
-                continue
+            internal.append(item)
            if item.endswith('.js'):
-                has_js = True
+                internal.has_js = True
            if item.endswith('.css'):
-                has_css = True
-            files.append(item)
+                internal.has_css = True            
+        if internal:
+            files.append(internal)

-        if have_minify and ((self.optimize_css and has_css) or (self.optimize_js and has_js)):
-            # cache for 5 minutes by default
-            key = hashlib.md5(repr(files)).hexdigest()
-
-            cache = self.cache_includes or (current.cache.ram, 60 * 5)
-
-            def call_minify(files=files):
-                return minify.minify(files,
-                                     URL('static', 'temp'),
-                                     current.request.folder,
-                                     self.optimize_css,
-                                     self.optimize_js)
-            if cache:
-                cache_model, time_expire = cache
-                files = cache_model('response.files.minified/' + key,
-                                    call_minify,
-                                    time_expire)
-            else:
-                files = call_minify()
-
-        files.extend(ext_files)
-        s = []
-        for item in files:
+        # We're done we can now minify
+        if have_minify:
+            for i, f in enumerate(files):
+                if isinstance(f, List) and ((self.optimize_css and f.has_css) or (self.optimize_js and f.has_js)):
+                    # cache for 5 minutes by default
+                    key = hashlib_md5(repr(f)).hexdigest()
+                    cache = self.cache_includes or (current.cache.ram, 60 * 5)
+                    def call_minify(files=f):
+                        return List(minify.minify(files,
+                                             URL('static', 'temp'),
+                                             current.request.folder,
+                                             self.optimize_css,
+                                             self.optimize_js))
+                    if cache:
+                        cache_model, time_expire = cache
+                        files[i] = cache_model('response.files.minified/' + key,
+                                            call_minify,
+                                            time_expire)
+                    else:
+                        files[i] = call_minify()
+        
+        def static_map(s, item):
            if isinstance(item, str):
                f = item.lower().split('?')[0]
                ext = f.rpartition('.')[2]
@@ -523,6 +557,14 @@ class Response(Storage):
                tmpl = template_mapping.get(f)
                if tmpl:
                    s.append(tmpl % item[1])
+
+        s = []
+        for item in files:
+            if isinstance(item, List):
+                for f in item:
+                    static_map(s, f)
+            else:
+                static_map(s, item)
        self.write(''.join(s), escape=False)

    def stream(self,
@@ -578,9 +620,9 @@ class Response(Storage):
        if hasattr(stream, 'name'):
            filename = stream.name

-        if filename and not 'content-type' in keys:
+        if filename and 'content-type' not in keys:
            headers['Content-Type'] = contenttype(filename)
-        if filename and not 'content-length' in keys:
+        if filename and 'content-length' not in keys:
            try:
                headers['Content-Length'] = \
                    os.path.getsize(filename)
@@ -638,6 +680,11 @@ class Response(Storage):
        if download_filename is None:
            download_filename = filename
        if attachment:
+            # Browsers still don't have a simple uniform way to have non ascii
+            # characters in the filename so for now we are percent encoding it
+            if isinstance(download_filename, unicodeT):
+                download_filename = download_filename.encode('utf-8')
+            download_filename = urllib_quote(download_filename)
            headers['Content-Disposition'] = \
                'attachment; filename="%s"' % download_filename.replace('"', '\"')
        return self.stream(stream, chunk_size=chunk_size, request=request)
@@ -1022,7 +1069,7 @@ class Session(Storage):
        if self._forget:
            del rcookies[response.session_id_name]
            return
-        if self.get('httponly_cookies',True):
+        if self.get('httponly_cookies', True):
            scookies['HttpOnly'] = True
        if self._secure:
            scookies['secure'] = True
@@ -1193,7 +1240,7 @@ class Session(Storage):
            if (not response.session_id or
                not response.session_filename or
                self._forget
-                or self._unchanged(response)):
+                    or self._unchanged(response)):
                # self.clear_session_cookies()
                return False
            else:
@@ -182,8 +182,8 @@ class Highlighter(object):
        )),
        'PYTHONMultilineString': (python_tokenizer,
                                  (('ENDMULTILINESTRING',
-                                  re.compile(r'.*?("""|\'\'\')',
-                                  re.DOTALL), 'color: darkred'), )),
+                                    re.compile(r'.*?("""|\'\'\')',
+                                               re.DOTALL), 'color: darkred'), )),
        'HTML': (html_tokenizer, (
            ('GOTOPYTHON', re.compile(r'\{\{'), 'color: red'),
            ('COMMENT', re.compile(r'<!--[^>]*-->|<!>'),
@@ -209,7 +209,7 @@ class Highlighter(object):
        mode = self.mode
        while i < len(data):
            for (token, o_re, style) in Highlighter.all_styles[mode][1]:
-                if not token in self.suppress_tokens:
+                if token not in self.suppress_tokens:
                    match = o_re.match(data, i)
                    if match:
                        if style:
@@ -221,7 +221,7 @@ class Highlighter(object):
                            new_mode = \
                                Highlighter.all_styles[mode][0](self,
                                                                token, match, style)
-                        if not new_mode is None:
+                        if new_mode is not None:
                            mode = new_mode
                        i += max(1, len(match.group()))
                        break
@@ -241,9 +241,9 @@ class Highlighter(object):
            style = self.styles[token]
        if self.span_style != style:
            if style != 'Keep':
-                if not self.span_style is None:
+                if self.span_style is not None:
                    self.output.append('</span>')
-                if not style is None:
+                if style is not None:
                    self.output.append('<span style="%s">' % style)
                self.span_style = style

@@ -260,7 +260,7 @@ def highlight(
 ):
    styles = styles or {}
    attributes = attributes or {}
-    if not 'CODE' in styles:
+    if 'CODE' not in styles:
        code_style = """
        font-size: 11px;
        font-family: Bitstream Vera Sans Mono,monospace;
@@ -272,7 +272,7 @@ def highlight(
        white-space: pre !important;\n"""
    else:
        code_style = styles['CODE']
-    if not 'LINENUMBERS' in styles:
+    if 'LINENUMBERS' not in styles:
        linenumbers_style = """
        font-size: 11px;
        font-family: Bitstream Vera Sans Mono,monospace;
@@ -283,7 +283,7 @@ def highlight(
        color: #A0A0A0;\n"""
    else:
        linenumbers_style = styles['LINENUMBERS']
-    if not 'LINEHIGHLIGHT' in styles:
+    if 'LINEHIGHLIGHT' not in styles:
        linehighlight_style = "background-color: #EBDDE2;"
    else:
        linehighlight_style = styles['LINEHIGHLIGHT']
@@ -333,8 +333,9 @@ def highlight(
                  == '_' and value])
    if fa:
        fa = ' ' + fa
-    return '<table%s><tr style="vertical-align:top;"><td style="min-width:40px; text-align: right;"><pre style="%s">%s</pre></td><td><pre style="%s">%s</pre></td></tr></table>'\
-        % (fa, linenumbers_style, numbers, code_style, code)
+    return '<table%s><tr style="vertical-align:top;">' \
+           '<td style="min-width:40px; text-align: right;"><pre style="%s">%s</pre></td>' \
+           '<td><pre style="%s">%s</pre></td></tr></table>' % (fa, linenumbers_style, numbers, code_style, code)


 if __name__ == '__main__':
@@ -342,5 +343,4 @@ if __name__ == '__main__':
    argfp = open(sys.argv[1])
    data = argfp.read()
    argfp.close()
-    print('<html><body>' + highlight(data, sys.argv[2])\
-        + '</body></html>')
+    print('<html><body>' + highlight(data, sys.argv[2]) + '</body></html>')
@@ -20,7 +20,8 @@ import urllib
 import base64
 from gluon import sanitizer, decoder
 import itertools
-from gluon._compat import reduce, pickle, copyreg, HTMLParser, name2codepoint, iteritems, unichr, unicodeT, urllib_quote, to_bytes, to_native, to_unicode, basestring, urlencode, implements_bool, text_type, long
+from gluon._compat import reduce, pickle, copyreg, HTMLParser, name2codepoint, iteritems, unichr, unicodeT, \
+    urllib_quote, to_bytes, to_native, to_unicode, basestring, urlencode, implements_bool, text_type, long
 from gluon.utils import local_html_escape
 import marshal

@@ -109,6 +110,7 @@ __all__ = [

 DEFAULT_PASSWORD_DISPLAY = '*' * 8

+
 def xmlescape(data, quote=True):
    """
    Returns an escaped string of the provided data
@@ -124,10 +126,9 @@ def xmlescape(data, quote=True):

    if not(isinstance(data, (text_type, bytes))):
        # i.e., integers
-        data=str(data)
+        data = str(data)
    data = to_bytes(data, 'utf8', 'xmlcharrefreplace')

-
    # ... and do the escaping
    data = local_html_escape(data, quote)
    return data
@@ -671,6 +672,7 @@ def XML_pickle(data):
    return XML_unpickle, (marshal.dumps(str(data)),)
 copyreg.pickle(XML, XML_pickle, XML_unpickle)

+
@implements_bool
 class DIV(XmlComponent):
    """
@@ -1309,8 +1311,10 @@ class HTML(DIV):
    tag = b'html'

    strict = b'<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">\n'
-    transitional = b'<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">\n'
-    frameset = b'<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Frameset//EN" "http://www.w3.org/TR/html4/frameset.dtd">\n'
+    transitional = \
+        b'<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">\n'
+    frameset = \
+        b'<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Frameset//EN" "http://www.w3.org/TR/html4/frameset.dtd">\n'
    html5 = b'<!DOCTYPE HTML>\n'

    def xml(self):
@@ -1861,7 +1865,7 @@ class INPUT(DIV):
                except:
                    import traceback
                    print(traceback.format_exc())
-                    msg = "Validation error, field:%s %s" % (name,validator)
+                    msg = "Validation error, field:%s %s" % (name, validator)
                    raise Exception(msg)
                if errors is not None:
                    self.vars[name] = value
@@ -1911,7 +1915,7 @@ class INPUT(DIV):
        name = self.attributes.get('_name', None)
        if name and hasattr(self, 'errors') \
                and self.errors.get(name, None) \
-                and self['hideerror'] != True:
+                and self['hideerror'] is not True:
            self['_class'] = (self['_class'] and self['_class'] + ' ' or '') + 'invalidinput'
            return DIV.xml(self) + DIV(
                DIV(
@@ -1979,7 +1983,6 @@ class OPTGROUP(DIV):


 class SELECT(INPUT):
-
    """
    Examples:

@@ -2014,7 +2017,7 @@ class SELECT(INPUT):
        if value is not None:
            if not self['_multiple']:
                for c in options:  # my patch
-                    if ((value is not None) and (str(c['_value']) == str(value))):
+                    if (value is not None) and (str(c['_value']) == str(value)):
                        c['_selected'] = 'selected'
                    else:
                        c['_selected'] = None
@@ -2024,7 +2027,7 @@ class SELECT(INPUT):
                else:
                    values = [str(value)]
                for c in options:  # my patch
-                    if ((value is not None) and (str(c['_value']) in values)):
+                    if (value is not None) and (str(c['_value']) in values):
                        c['_selected'] = 'selected'
                    else:
                        c['_selected'] = None
@@ -2390,7 +2393,6 @@ class FORM(DIV):


 class BEAUTIFY(DIV):
-
    """
    Turns any list, dictionary, etc into decent looking html.

@@ -2429,7 +2431,7 @@ class BEAUTIFY(DIV):
        if level == 0:
            return
        for c in self.components:
-            if hasattr(c, 'value') and not callable(c.value):
+            if hasattr(c, 'value') and not callable(c.value) and not isinstance(c, cgi.FieldStorage):
                if c.value:
                    components.append(c.value)
            if hasattr(c, 'xml') and callable(c.xml):
@@ -2547,7 +2549,7 @@ class MENU(DIV):
                        li['_class'] = li['_class'] + ' ' + self['li_active']
                    else:
                        li['_class'] = self['li_active']
-                if len(item) <= 4 or item[4] == True:
+                if len(item) <= 4 or item[4] is True:
                    ul.append(li)
        return ul

@@ -2561,7 +2563,7 @@ class MENU(DIV):
                # ex: ('', False, A('title', _href=URL(...), _title="title"))
                # ex: (A('title', _href=URL(...), _title="title"), False, None)
                custom_items.append(item)
-            elif len(item) <= 4 or item[4] == True:
+            elif len(item) <= 4 or item[4] is True:
                select.append(OPTION(CAT(prefix, item[0]),
                                     _value=item[2], _selected=item[1]))
                if len(item) > 3 and len(item[3]):
@@ -2703,7 +2705,8 @@ class web2pyHTMLParser(HTMLParser):
                self.parent = self.parent.parent
            except:
                raise RuntimeError("unable to balance tag %s" % tagname)
-            if parent_tagname[:len(tagname)] == tagname: break
+            if parent_tagname[:len(tagname)] == tagname:
+                break


 def markdown_serializer(text, tag=None, attr=None):
@@ -11,7 +11,7 @@ HTTP statuses helpers
 """

 import re
-from gluon._compat import iteritems
+from gluon._compat import iteritems, unicodeT, to_bytes

 __all__ = ['HTTP', 'redirect']

@@ -111,22 +111,29 @@ class HTTP(Exception):
            if not body:
                body = status
            if isinstance(body, (str, bytes, bytearray)):
+                if isinstance(body, unicodeT):
+                    body = to_bytes(body) # This must be done before len
                headers['Content-Length'] = len(body)
        rheaders = []
        for k, v in iteritems(headers):
            if isinstance(v, list):
                rheaders += [(k, str(item)) for item in v]
-            elif not v is None:
+            elif v is not None:
                rheaders.append((k, str(v)))
        responder(status, rheaders)
        if env.get('request_method', '') == 'HEAD':
            return ['']
        elif isinstance(body, (str, bytes, bytearray)):
+            if isinstance(body, unicodeT):
+                body = to_bytes(body)
            return [body]
        elif hasattr(body, '__iter__'):
            return body
        else:
-            return [str(body)]
+            body = str(body)
+            if isinstance(body, unicodeT):
+                body = to_bytes(body)
+            return [body]

    @property
    def message(self):
@@ -148,7 +155,7 @@ class HTTP(Exception):
            web2py_error=self.headers.get('web2py_error'))

    def __str__(self):
-        "stringify me"
+        """stringify me"""
        return self.message


@@ -78,13 +78,9 @@ alert_dependency = ['hashlib', 'uuid']
 # Now we remove the blacklisted modules if we are using the stated
 # python version.
 #
-# List of modules deprecated in Python 2.6 or 2.7 that are in the above set
+# List of modules deprecated in Python 2.7 that are in the above list
 py27_deprecated = ['mhlib', 'multifile', 'mimify', 'sets', 'MimeWriter']  # And ['optparse'] but we need it for now

-if python_version >= '2.6':
-    base_modules += ['json', 'multiprocessing']
-    base_modules = list(set(base_modules).difference(set(py26_deprecated)))
-
 if python_version >= '2.7':
    base_modules += ['argparse', 'json', 'multiprocessing']
    base_modules = list(set(base_modules).difference(set(py27_deprecated)))
@@ -18,10 +18,11 @@ import pkgutil
 import logging
 from cgi import escape
 from threading import RLock
-from gluon.utf8 import Utf8
+
 from gluon.utils import local_html_escape

 from gluon._compat import copyreg, PY2, maketrans, iterkeys, unicodeT, to_unicode, to_bytes, iteritems, to_native, pjoin
+
 from pydal.contrib.portalocker import read_locked, LockedFile

 from gluon.fileutils import listdir
@@ -49,8 +50,10 @@ DEFAULT_CONSTRUCT_PLURAL_FORM = lambda word, plural_id: word

 if PY2:
    NUMBERS = (int, long, float)
+    from gluon.utf8 import Utf8
 else:
    NUMBERS = (int, float)
+    Utf8 = str

 # pattern to find T(blah blah blah) expressions
 PY_STRING_LITERAL_RE = r'(?<=[^\w]T\()(?P<name>'\
@@ -107,15 +110,17 @@ def markmin(s):


 def upper_fun(s):
-    return unicode(s, 'utf-8').upper().encode('utf-8')
+    return to_unicode(s).upper()


 def title_fun(s):
-    return unicode(s, 'utf-8').title().encode('utf-8')
+    return to_unicode(s).title()


 def cap_fun(s):
-    return unicode(s, 'utf-8').capitalize().encode('utf-8')
+    return to_unicode(s).capitalize()
+
+
 ttab_in = maketrans("\\%{}", '\x1c\x1d\x1e\x1f')
 ttab_out = maketrans('\x1c\x1d\x1e\x1f', "\\%{}")

@@ -11,7 +11,8 @@ The gluon wsgi application
 """
 from __future__ import print_function

-if False: import import_all # DO NOT REMOVE PART OF FREEZE PROCESS
+if False:
+    import import_all  # DO NOT REMOVE PART OF FREEZE PROCESS
 import gc

 import os
@@ -26,7 +27,7 @@ import random
 import string

 from gluon._compat import Cookie, urllib2
-#from thread import allocate_lock
+# from thread import allocate_lock

 from gluon.fileutils import abspath, write_file
 from gluon.settings import global_settings
@@ -67,14 +68,14 @@ import gluon.messageboxhandler
 logging.gluon = gluon
 # so we must restore it! Thanks ozancag
 import locale
-locale.setlocale(locale.LC_CTYPE, "C") # IMPORTANT, web2py requires locale "C"
+locale.setlocale(locale.LC_CTYPE, "C")  # IMPORTANT, web2py requires locale "C"

 exists = os.path.exists
 pjoin = os.path.join

 try:
    logging.config.fileConfig(abspath("logging.conf"))
-except: # fails on GAE or when logfile is missing
+except:  # fails on GAE or when logfile is missing
    logging.basicConfig()
 logger = logging.getLogger("web2py")

@@ -230,7 +231,7 @@ class LazyWSGI(object):

        to call third party WSGI applications
        """
-        self.response.status = str(status).split(' ', 1)[0]
+        self.response.status = int(str(status).split(' ', 1)[0])
        self.response.headers = dict(headers)
        return lambda *args, **kargs: \
            self.response.write(escape=False, *args, **kargs)
@@ -254,6 +255,7 @@ class LazyWSGI(object):
                return [data]
            for item in middleware_apps:
                app = item(app)
+
            def caller(app):
                return app(self.environ, self.start_response)
            return lambda caller=caller, app=app: caller(app)
@@ -294,9 +296,9 @@ def wsgibase(environ, responder):
    response = Response()
    session = Session()
    env = request.env
-    #env.web2py_path = global_settings.applications_parent
+    # env.web2py_path = global_settings.applications_parent
    env.web2py_version = web2py_version
-    #env.update(global_settings)
+    # env.update(global_settings)
    static_file = False
    http_response = None
    try:
@@ -325,7 +327,6 @@ def wsgibase(environ, responder):
                            'Expires'] = 'Thu, 31 Dec 2037 23:59:59 GMT'
                    response.stream(static_file, request=request)

-
                # ##################################################
                # fill in request items
                # ##################################################
@@ -356,17 +357,15 @@ def wsgibase(environ, responder):
                cmd_opts = global_settings.cmd_options

                request.update(
-                    client = client,
-                    folder = abspath('applications', app) + os.sep,
-                    ajax = x_req_with == 'xmlhttprequest',
-                    cid = env.http_web2py_component_element,
-                    is_local = (env.remote_addr in local_hosts and
-                                client == env.remote_addr),
-                    is_shell = False,
-                    is_scheduler = False,
-                    is_https = env.wsgi_url_scheme in HTTPS_SCHEMES or \
-                        request.env.http_x_forwarded_proto in HTTPS_SCHEMES \
-                        or env.https == 'on'
+                    client=client,
+                    folder=abspath('applications', app) + os.sep,
+                    ajax=x_req_with == 'xmlhttprequest',
+                    cid=env.http_web2py_component_element,
+                    is_local=(env.remote_addr in local_hosts and client == env.remote_addr),
+                    is_shell=False,
+                    is_scheduler=False,
+                    is_https=env.wsgi_url_scheme in HTTPS_SCHEMES or
+                             request.env.http_x_forwarded_proto in HTTPS_SCHEMES or env.https == 'on'
                    )
                request.url = environ['PATH_INFO']

@@ -390,7 +389,7 @@ def wsgibase(environ, responder):
                                   % 'invalid request',
                                   web2py_error='invalid application')
                elif not request.is_local and exists(disabled):
-                    five0three = os.path.join(request.folder,'static','503.html')
+                    five0three = os.path.join(request.folder, 'static', '503.html')
                    if os.path.exists(five0three):
                        raise HTTP(503, file(five0three, 'r').read())
                    else:
@@ -406,7 +405,7 @@ def wsgibase(environ, responder):
                # get the GET and POST data
                # ##################################################

-                #parse_get_post_vars(request, environ)
+                # parse_get_post_vars(request, environ)

                # ##################################################
                # expose wsgi hooks for convenience
@@ -625,7 +624,7 @@ def appfactory(wsgiapp=wsgibase,
                raise BaseException("Can't create dir %s" % profiler_dir)
        filepath = pjoin(profiler_dir, 'wtest')
        try:
-            filehandle = open( filepath, 'w' )
+            filehandle = open(filepath, 'w')
            filehandle.close()
            os.unlink(filepath)
        except IOError:
@@ -746,7 +745,7 @@ class HttpServer(object):
        sock_list = [ip, port]
        if not ssl_certificate or not ssl_private_key:
            logger.info('SSL is off')
-        elif not rocket.ssl:
+        elif not rocket.has_ssl:
            logger.warning('Python "ssl" module unavailable. SSL is OFF')
        elif not exists(ssl_certificate):
            logger.warning('unable to open SSL certificate. SSL is OFF')
@@ -225,7 +225,7 @@ def parsecronline(line):
    params = line.strip().split(None, 6)
    if len(params) < 7:
        return None
-    daysofweek = {'sun': 0, 'mon': 1, 'tue': 2, 'wed': 3, 
+    daysofweek = {'sun': 0, 'mon': 1, 'tue': 2, 'wed': 3,
                  'thu': 4, 'fri': 5, 'sat': 6}
    for (s, id) in zip(params[:5], ['min', 'hr', 'dom', 'mon', 'dow']):
        if not s in [None, '*']:
@@ -10,7 +10,7 @@ Restricted environment to execute application's code
 """

 import sys
-from gluon._compat import pickle, ClassType
+from gluon._compat import pickle, ClassType, unicodeT, to_bytes
 import traceback
 import types
 import os
@@ -192,10 +192,10 @@ class RestrictedError(Exception):
        # safely show an useful message to the user
        try:
            output = self.output
-            if isinstance(output, unicode):
-                output = output.encode("utf8")
-            elif not isinstance(output, str):
+            if not isinstance(output, str, bytes, bytearray):
                output = str(output)
+            if isinstance(output, unicodeT):
+                output = to_bytes(output)
        except:
            output = ""
        return output
@@ -205,7 +205,7 @@ def compile2(code, layer):
    return compile(code, layer, 'exec')


-def restricted(ccode, environment=None, layer='Unknown'):
+def restricted(ccode, environment=None, layer='Unknown', scode=None):
    """
    Runs code in environment and returns the output. If an exception occurs
    in code it raises a RestrictedError containing the traceback. Layer is
@@ -230,7 +230,9 @@ def restricted(ccode, environment=None, layer='Unknown'):
            sys.excepthook(etype, evalue, tb)
        del tb
        output = "%s %s" % (etype, evalue)
-        raise RestrictedError(layer, ccode, output, environment)
+        # Save source code in ticket when available
+        scode = scode if scode else ccode
+        raise RestrictedError(layer, scode, output, environment)


 def snapshot(info=None, context=5, code=None, environment=None):
@@ -206,8 +206,7 @@ def url_out(request, environ, application, controller, function,
    if host is True or (host is None and (scheme or port is not None)):
        host = request.env.http_host
    if not scheme or scheme is True:
-        scheme = request.env.get('wsgi_url_scheme', 'http').lower() \
-            if request else 'http'
+        scheme = request.env.get('wsgi_url_scheme', 'http').lower() if request else 'http'
    if host:
        host_port = host if not port else host.split(':', 1)[0] + ':%s' % port
        url = '%s://%s%s' % (scheme, host_port, url)
@@ -145,7 +145,7 @@ class XssCleaner(HTMLParser):
        if url.startswith('#'):
            return True
        else:
-            parsed = urlparse(url)
+            parsed = urlparse.urlparse(url)
            return ((parsed[0] in self.allowed_schemes and '.' in parsed[1]) or
                    (parsed[0] in self.allowed_schemes and '@' in parsed[2]) or
                    (parsed[0] == '' and parsed[2].startswith('/')))
@@ -119,8 +119,8 @@ def xml(value, encoding='UTF-8', key='document', quote=True):
    return ('<?xml version="1.0" encoding="%s"?>' % encoding) + str(xml_rec(value, key, quote))


-def json(value, default=custom_json, indent=None):
-    value = json_parser.dumps(value, default=default, sort_keys=True, indent=indent)
+def json(value, default=custom_json, indent=None, sort_keys=False):
+    value = json_parser.dumps(value, default=default, sort_keys=sort_keys, indent=indent)
    # replace JavaScript incompatible spacing
    # http://timelessrepo.com/json-isnt-a-javascript-subset
    # PY3 FIXME
@@ -31,10 +31,16 @@ from gluon.globals import Request, Response, Session
 from gluon.storage import Storage, List
 from gluon.admin import w2p_unpack
 from pydal.base import BaseAdapter
-from gluon._compat import iteritems, ClassType
+from gluon._compat import iteritems, ClassType, PY2

 logger = logging.getLogger("web2py")

+if not PY2:
+    def execfile(filename, global_vars=None, local_vars=None):
+        with open(filename) as f:
+            code = compile(f.read(), filename, 'exec')
+            exec(code, global_vars, local_vars)
+

 def enable_autocomplete_and_history(adir, env):
    try:
@@ -17,6 +17,7 @@ Holds:
 import datetime
 import urllib
 import re
+import copy

 import os
 from gluon._compat import StringIO, unichr, urllib_quote, iteritems, basestring, long, unicodeT, to_native, to_unicode
@@ -405,7 +406,7 @@ class RadioWidget(OptionsWidget):
        cols = attributes.get('cols', 1)
        totals = len(options)
        mods = totals % cols
-        rows = totals / cols
+        rows = totals // cols
        if mods:
            rows += 1

@@ -471,7 +472,7 @@ class CheckboxesWidget(OptionsWidget):
        cols = attributes.get('cols', 1)
        totals = len(options)
        mods = totals % cols
-        rows = totals / cols
+        rows = totals // cols
        if mods:
            rows += 1

@@ -658,7 +659,8 @@ class AutocompleteWidget(object):
                 orderby=None, limitby=(0, 10), distinct=False,
                 keyword='_autocomplete_%(tablename)s_%(fieldname)s',
                 min_length=2, help_fields=None, help_string=None,
-                 at_beginning=True, default_var='ac'):
+                 at_beginning=True, default_var='ac', user_signature=True,
+                 hash_vars=False):

        self.help_fields = help_fields or []
        self.help_string = help_string
@@ -681,12 +683,14 @@ class AutocompleteWidget(object):
        else:
            self.is_reference = False
        if hasattr(request, 'application'):
-            urlvars = request.vars
+            urlvars = copy.copy(request.vars)
            urlvars[default_var] = 1
-            self.url = URL(args=request.args, vars=urlvars)
-            self.callback()
+            self.url = URL(args=request.args, vars=urlvars,
+                           user_signature=user_signature, hash_vars=hash_vars)
+            self.run_callback = True
        else:
            self.url = request
+            self.run_callback = False

    def callback(self):
        if self.keyword in self.request.vars:
@@ -759,6 +763,8 @@ class AutocompleteWidget(object):
                raise HTTP(200, '')

    def __call__(self, field, value, **attributes):
+        if self.run_callback:
+            self.callback()
        default = dict(
            _type='text',
            value=(value is not None and str(value)) or '',
@@ -1082,6 +1088,106 @@ def formstyle_bootstrap3_inline_factory(col_label_size=3):
        return parent
    return _inner

+# bootstrap 4
+def formstyle_bootstrap4_stacked(form, fields):
+    """ bootstrap 3 format form layout
+
+    Note:
+        Experimental!
+    """
+    parent = CAT()
+    for id, label, controls, help in fields:
+        # wrappers
+        _help = SPAN(help, _class='help-block')
+        # embed _help into _controls
+        _controls = CAT(controls, _help)
+        if isinstance(controls, INPUT):
+            if controls['_type'] == 'submit':
+                controls.add_class('btn btn-primary')
+            if controls['_type'] == 'button':
+                controls.add_class('btn btn-secondary')
+            elif controls['_type'] == 'file':
+                controls.add_class('input-file')
+            elif controls['_type'] in ('text', 'password'):
+                controls.add_class('form-control')
+            elif controls['_type'] == 'checkbox':
+                label['_for'] = None
+                label.insert(0, controls)
+                label.insert(0, ' ')
+                _controls = DIV(label, _help, _class="checkbox")
+                label = ''
+            elif isinstance(controls, (SELECT, TEXTAREA)):
+                controls.add_class('form-control')
+
+        elif isinstance(controls, SPAN):
+            _controls = P(controls.components)
+
+        elif isinstance(controls, UL):
+            for e in controls.elements("input"):
+                e.add_class('form-control')
+
+        elif isinstance(controls, CAT) and isinstance(controls[0], INPUT):
+            controls[0].add_class('form-control')
+
+        if isinstance(label, LABEL):
+            label['_class'] = add_class(label.get('_class'), 'form-control-label')
+
+        parent.append(DIV(label, _controls, _class='form-group', _id=id))
+    return parent
+
+
+def formstyle_bootstrap4_inline_factory(col_label_size=3):
+    """ bootstrap 3 horizontal form layout
+
+    Note:
+        Experimental!
+    """
+    def _inner(form, fields):
+        form.add_class('form-horizontal')
+        label_col_class = "col-sm-%d" % col_label_size
+        col_class = "col-sm-%d" % (12 - col_label_size)
+        offset_class = "col-sm-offset-%d" % col_label_size
+        parent = CAT()
+        for id, label, controls, help in fields:
+            # wrappers
+            _help = SPAN(help, _class='help-block')
+            # embed _help into _controls
+            _controls = DIV(controls, _help, _class="%s" % (col_class))
+            if isinstance(controls, INPUT):
+                if controls['_type'] == 'submit':
+                    controls.add_class('btn btn-primary')
+                    _controls = DIV(controls, _class="%s %s" % (col_class, offset_class))
+                if controls['_type'] == 'button':
+                    controls.add_class('btn btn-secondary')
+                elif controls['_type'] == 'file':
+                    controls.add_class('input-file')
+                elif controls['_type'] in ('text', 'password'):
+                    controls.add_class('form-control')
+                elif controls['_type'] == 'checkbox':
+                    label['_for'] = None
+                    label.insert(0, controls)
+                    label.insert(1, ' ')
+                    _controls = DIV(DIV(label, _help, _class="checkbox"),
+                                    _class="%s %s" % (offset_class, col_class))
+                    label = ''
+                elif isinstance(controls, (SELECT, TEXTAREA)):
+                    controls.add_class('form-control')
+
+            elif isinstance(controls, SPAN):
+                _controls = P(controls.components,
+                              _class="form-control-static %s" % col_class)
+            elif isinstance(controls, UL):
+                for e in controls.elements("input"):
+                    e.add_class('form-control')
+            elif isinstance(controls, CAT) and isinstance(controls[0], INPUT):
+                    controls[0].add_class('form-control')
+            if isinstance(label, LABEL):
+                label['_class'] = add_class(label.get('_class'), 'form-control-label %s' % label_col_class)
+
+            parent.append(DIV(label, _controls, _class='form-group', _id=id))
+        return parent
+    return _inner
+

 class SQLFORM(FORM):

@@ -1170,6 +1276,8 @@ class SQLFORM(FORM):
        bootstrap=formstyle_bootstrap,
        bootstrap3_stacked=formstyle_bootstrap3_stacked,
        bootstrap3_inline=formstyle_bootstrap3_inline_factory(3),
+        bootstrap4_stacked=formstyle_bootstrap4_stacked,
+        bootstrap4_inline=formstyle_bootstrap4_inline_factory(3),
        inline=formstyle_inline,
        )

@@ -1238,8 +1346,8 @@ class SQLFORM(FORM):
        # will only use writable or readable fields, unless forced to ignore
        if fields is None:
            fields = [f.name for f in table if
-                      (ignore_rw or f.writable or f.readable) and
-                      (readonly or not f.compute)]
+                      (ignore_rw or f.writable or f.readable) and 
+                      not (f.compute and not record)]
        self.fields = fields

        # make sure we have an id
@@ -1322,7 +1430,7 @@ class SQLFORM(FORM):
            label = LABEL(label, label and sep, _for=field_id,
                          _id=field_id + SQLFORM.ID_LABEL_SUFFIX)

-            cond = readonly or \
+            cond = readonly or field.compute or \
                (not ignore_rw and not field.writable and field.readable)

            if cond:
@@ -1916,8 +2024,10 @@ class SQLFORM(FORM):
        if 'table_name' in attributes:
            del attributes['table_name']

-        return SQLFORM(DAL(None).define_table(table_name, *fields),
-                       **attributes)
+        # Clone fields, while passing tables straight through
+        fields_with_clones = [f.clone() if isinstance(f, Field) else f for f in fields]
+
+        return SQLFORM(DAL(None).define_table(table_name, *fields_with_clones), **attributes)

    @staticmethod
    def build_query(fields, keywords):
@@ -1932,11 +2042,13 @@ class SQLFORM(FORM):
            if settings.global_settings.web2py_runtime_gae:
                return reduce(lambda a,b: a|b, [field.contains(key) for field in sfields])
            else:
+                if not (sfields and key and key.split()):
+                    return fields[0].table
                return reduce(lambda a,b:a&b,[
                        reduce(lambda a,b: a|b, [
                                field.contains(k) for field in sfields]
                               ) for k in key.split()])
-
+                    
            # from https://groups.google.com/forum/#!topic/web2py/hKe6lI25Bv4
            # needs testing...
            #words = key.split(' ') if key else []
@@ -2156,6 +2268,7 @@ class SQLFORM(FORM):
             represent_none=None,
             showblobs=False):

+        dbset = None
        formstyle = formstyle or current.response.formstyle
        if isinstance(query, Set):
            query = query.query
@@ -2337,12 +2450,12 @@ class SQLFORM(FORM):
                for k, f in iteritems(table):
                    if isinstance(f, Field.Virtual):
                        f.tablename = table._tablename
-            columns = [f for f in fields if f.tablename in tablenames]
+            columns = [f for f in fields if f.tablename in tablenames and f.listable]
        else:
            fields = []
            columns = []
            filter1 = lambda f: isinstance(f, Field) and (f.type!='blob' or showblobs)
-            filter2 = lambda f: isinstance(f, Field) and f.readable
+            filter2 = lambda f: isinstance(f, Field) and f.readable and f.listable
            for table in tables:
                fields += filter(filter1, table)
                columns += filter(filter2, table)
@@ -2560,8 +2673,8 @@ class SQLFORM(FORM):
                    try:
                        # the query should be constructed using searchable
                        # fields but not virtual fields
-                        sfields = reduce(lambda a, b: a + b,
-                            [[f for f in t if f.readable and not isinstance(f, Field.Virtual)] for t in tables])
+                        is_searchable = lambda f: f.readable and not isinstance(f, Field.Virtual) and f.searchable
+                        sfields = reduce(lambda a, b: a + b, [filter(is_searchable, t) for t in tables])
                        # use custom_query using searchable
                        if callable(searchable):
                            dbset = dbset(searchable(sfields, keywords))
@@ -3034,6 +3147,7 @@ class SQLFORM(FORM):
        res.view_form = view_form
        res.search_form = search_form
        res.rows = rows
+        res.dbset = dbset
        return res

    @staticmethod
@@ -3152,8 +3266,8 @@ class SQLFORM(FORM):
                # if isinstance(linked_tables, dict):
                #     linked_tables = linked_tables.get(table._tablename, [])
                if linked_tables is None or referee in linked_tables:
-                    field.represent = (lambda id, r=None, referee=referee, rep=field.represent: 
-                                       A(callable(rep) and rep(id) or id, 
+                    field.represent = (lambda id, r=None, referee=referee, rep=field.represent:
+                                       A(callable(rep) and rep(id) or id,
                                         cid=request.cid, _href=url(args=['view', referee, id])))
        except (KeyError, ValueError, TypeError):
            redirect(URL(args=table._tablename))
@@ -3316,7 +3430,7 @@ class SQLTABLE(TABLE):
            return
        REGEX_TABLE_DOT_FIELD = sqlrows.db._adapter.REGEX_TABLE_DOT_FIELD
        fieldmap = dict(zip(sqlrows.colnames, sqlrows.fields))
-        tablemap = dict(((f.tablename, f.table) for f in fieldmap.values()))
+        tablemap = dict(((f.tablename, f.table) if isinstance(f, Field) else (f._table._tablename, f._table) for f in fieldmap.values()))
        for table in tablemap.values():
            pref = table._tablename + '.'
            fieldmap.update(((pref+f.name, f) for f in table._virtual_fields))
@@ -3559,7 +3673,9 @@ class ExportClass(object):
                if not self.rows.db._adapter.REGEX_TABLE_DOT_FIELD.match(col):
                    row.append(record._extra[col])
                else:
-                    (t, f) = col.split('.')
+                    # The grid code modifies rows.colnames, adding double quotes
+                    # around the table and field names -- so they must be removed here.
+                    (t, f) = [name.strip('"') for name in col.split('.')]
                    field = self.rows.db[t][f]
                    if isinstance(record.get(t, None), (Row, dict)):
                        value = record[t][f]
@@ -17,6 +17,7 @@ from gluon import fileutils
 from gluon.dal import DAL, Field, Table
 from gluon.http import HTTP
 from gluon.fileutils import open_file
+from gluon.cache import CacheInRam

 DEFAULT_URI = os.getenv('DB', 'sqlite:memory')

@@ -104,6 +105,21 @@ class TestAppAdmin(unittest.TestCase):
        self._test_index()
        remove_compiled_application(appname_path)

+    def test_index_minify(self):
+        # test for gluon/contrib/minify
+        self.env['response'].optimize_css = 'concat|minify'
+        self.env['response'].optimize_js = 'concat|minify'
+        self.env['current'].cache = Storage({'ram':CacheInRam()})
+        appname_path = os.path.join(os.getcwd(), 'applications', 'welcome')
+        self._test_index()
+        file_l = os.listdir(os.path.join(appname_path, 'static', 'temp'))
+        file_l.sort()
+        self.assertTrue(len(file_l) == 2)
+        self.assertEqual(file_l[0][0:10], 'compressed')
+        self.assertEqual(file_l[1][0:10], 'compressed')
+        self.assertEqual(file_l[0][-3:], 'css')
+        self.assertEqual(file_l[1][-2:], 'js')
+
    def test_select(self):
        request = self.env['request']
        request.args = List(['db'])
@@ -138,6 +138,9 @@ class TestAuthAPI(unittest.TestCase):
        self.assertTrue('new_password2' in result['errors'])
        result = self.auth.change_password(old_password='bart_password', new_password='1234', new_password2='1234')
        self.assertTrue('old_password' in result['errors'])
+        # Test the default 4 min_length is enforced on change password
+        result = self.auth.change_password(old_password='1234', new_password='123', new_password2='123')
+        self.assertTrue('new_password' in result['errors'])

    def test_verify_key(self):
        self.auth.settings.registration_requires_verification = True
@@ -1,10 +1,11 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-

+import os
 import unittest
 import datetime

-from gluon.fileutils import parse_version
+from gluon.fileutils import parse_version, fix_newlines


 class TestFileUtils(unittest.TestCase):
@@ -22,3 +23,6 @@ class TestFileUtils(unittest.TestCase):
        # Semantic Beta
        rtn = parse_version('Version 2.14.1-beta+timestamp.2016.03.21.22.35.26')
        self.assertEqual(rtn, (2, 14, 1, 'beta', datetime.datetime(2016, 3, 21, 22, 35, 26)))
+    
+    def test_fix_newlines(self):
+        fix_newlines(os.path.dirname(os.path.abspath(__file__)))
@@ -158,10 +158,10 @@ class testResponse(unittest.TestCase):
        response.files.append(URL('a', 'static', 'css/file.ts'))
        content = return_includes(response)
        self.assertEqual(content,
+                         '<script src="http://maps.google.com/maps/api/js?sensor=false" type="text/javascript"></script>' +
                         '<script src="https://code.jquery.com/jquery-1.11.3.min.js?var=0" type="text/javascript"></script>' +
                         '<link href="/a/static/css/file.css" rel="stylesheet" type="text/css" />' +
-                         '<script src="/a/static/css/file.ts" type="text/typescript"></script>' +
-                         '<script src="http://maps.google.com/maps/api/js?sensor=false" type="text/javascript"></script>'
+                         '<script src="/a/static/css/file.ts" type="text/typescript"></script>'
                         )

        response = Response()
@@ -355,10 +355,10 @@ class TestSQLTABLE(unittest.TestCase):

        self.db.commit()

-#    def test_SQLTABLE(self):
-#        rows = self.db(self.db.auth_user.id > 0).select(self.db.auth_user.ALL)
-#        sqltable = SQLTABLE(rows)
-#        self.assertEqual(sqltable.xml(), '<table><thead><tr><th>auth_user.id</th><th>auth_user.first_name</th><th>auth_user.last_name</th><th>auth_user.email</th><th>auth_user.username</th><th>auth_user.password</th><th>auth_user.registration_key</th><th>auth_user.reset_password_key</th><th>auth_user.registration_id</th></tr></thead><tbody><tr class="w2p_odd odd"><td>1</td><td>Bart</td><td>Simpson</td><td>user1@test.com</td><td>user1</td><td>password_123</td><td>None</td><td></td><td>None</td></tr></tbody></table>')
+    def test_SQLTABLE(self):
+        rows = self.db(self.db.auth_user.id > 0).select(self.db.auth_user.ALL)
+        sqltable = SQLTABLE(rows)
+        self.assertEqual(sqltable.xml(), b'<table><thead><tr><th>auth_user.id</th><th>auth_user.first_name</th><th>auth_user.last_name</th><th>auth_user.email</th><th>auth_user.username</th><th>auth_user.password</th><th>auth_user.registration_key</th><th>auth_user.reset_password_key</th><th>auth_user.registration_id</th></tr></thead><tbody><tr class="w2p_odd odd"><td>1</td><td>Bart</td><td>Simpson</td><td>user1@test.com</td><td>user1</td><td>password_123</td><td>None</td><td></td><td>None</td></tr></tbody></table>')


 # class TestExportClass(unittest.TestCase):
@@ -278,8 +278,68 @@ class TestValidators(unittest.TestCase):
        self.assertEqual(rtn, ('jerry', 'oops'))
        rtn = IS_IN_DB(db, 'person.id', '%(name)s', auto_add=True)('jerry')
        self.assertEqual(rtn, (3, None))
+        # Test it works with reference table
+        db.define_table('ref_table',
+                        Field('name'),
+                        Field('person_id', 'reference person')
+                        )
+        ret = db.ref_table.validate_and_insert(name='test reference table')
+        self.assertFalse(list(ret.errors))
+        ret = db.ref_table.validate_and_insert(name='test reference table', person_id=george_id)
+        self.assertFalse(list(ret.errors))
+        rtn = IS_IN_DB(db, 'ref_table.person_id', '%(name)s')(george_id)
+        self.assertEqual(rtn, (george_id, None))
+        # Test it works with reference table.field and keyed table
+        db.define_table('person_keyed',
+                        Field('name'),
+                        primarykey=['name'])
+        db.person_keyed.insert(name='george')
+        db.person_keyed.insert(name='costanza')
+        rtn = IS_IN_DB(db, 'person_keyed.name')('george')
+        self.assertEqual(rtn, ('george', None))
+        db.define_table('ref_table_field',
+                        Field('name'),
+                        Field('person_name', 'reference person_keyed.name')
+                        )
+        ret = db.ref_table_field.validate_and_insert(name='test reference table.field')
+        self.assertFalse(list(ret.errors))
+        ret = db.ref_table_field.validate_and_insert(name='test reference table.field', person_name='george')
+        self.assertFalse(list(ret.errors))
+        vldtr = IS_IN_DB(db, 'ref_table_field.person_name', '%(name)s')
+        vldtr.options()
+        rtn = vldtr('george')
+        self.assertEqual(rtn, ('george', None))
+        # Test it works with list:reference table
+        db.define_table('list_ref_table',
+                        Field('name'),
+                        Field('person_list', 'list:reference person'))
+        ret = db.list_ref_table.validate_and_insert(name='test list:reference table')
+        self.assertFalse(list(ret.errors))
+        ret = db.list_ref_table.validate_and_insert(name='test list:reference table', person_list=[george_id,costanza_id])
+        self.assertFalse(list(ret.errors))
+        vldtr = IS_IN_DB(db, 'list_ref_table.person_list')
+        vldtr.options()
+        rtn = vldtr([george_id,costanza_id])
+        self.assertEqual(rtn, ([george_id,costanza_id], None))
+        # Test it works with list:reference table.field and keyed table
+        #db.define_table('list_ref_table_field',
+        #                Field('name'),
+        #                Field('person_list', 'list:reference person_keyed.name'))
+        #ret = db.list_ref_table_field.validate_and_insert(name='test list:reference table.field')
+        #self.assertFalse(list(ret.errors))
+        #ret = db.list_ref_table_field.validate_and_insert(name='test list:reference table.field', person_list=['george','costanza'])
+        #self.assertFalse(list(ret.errors))
+        #vldtr = IS_IN_DB(db, 'list_ref_table_field.person_list')
+        #vldtr.options()
+        #rtn = vldtr(['george','costanza'])
+        #self.assertEqual(rtn, (['george','costanza'], None))
        db.person.drop()
        db.category.drop()
+        db.person_keyed.drop()
+        db.ref_table.drop()
+        db.ref_table_field.drop()
+        db.list_ref_table.drop()
+        #db.list_ref_table_field.drop()

    def test_IS_NOT_IN_DB(self):
        from gluon.dal import DAL, Field
@@ -444,6 +504,8 @@ class TestValidators(unittest.TestCase):
        self.assertEqual(rtn, (None, 'Enter a value'))
        rtn = IS_NOT_EMPTY()('')
        self.assertEqual(rtn, ('', 'Enter a value'))
+        rtn = IS_NOT_EMPTY()(b'')
+        self.assertEqual(rtn, (b'', 'Enter a value'))
        rtn = IS_NOT_EMPTY()('  ')
        self.assertEqual(rtn, ('  ', 'Enter a value'))
        rtn = IS_NOT_EMPTY()(' \n\t')
@@ -102,12 +102,12 @@ class TestWeb(LiveTest):
                    password='test',
                    _formname='login')
        client.post('user/login', data=data)
-        self.assertTrue('Welcome Homer' in client.text)
+        self.assertTrue('Homer' in client.text)

        # check registration and login were successful
        client.get('index')

-        self.assertTrue('Welcome Homer' in client.text)
+        self.assertTrue('Homer' in client.text)

        client = WebClient('http://127.0.0.1:8000/admin/default/')
        client.post('index', data=dict(password='testpass'))
@@ -1127,7 +1127,6 @@ def addrow(form, a, b, c, style, _id, position=-1):


 class AuthJWT(object):
-
    """
    Experimental!

@@ -1321,7 +1320,7 @@ class AuthJWT(object):
        # is the following safe or should we use
        # calendar.timegm(datetime.datetime.utcnow().timetuple())
        # result seem to be the same (seconds since epoch, in UTC)
-        now = time.mktime(datetime.datetime.now().timetuple())
+        now = time.mktime(datetime.datetime.utcnow().timetuple())
        expires = now + self.expiration
        payload = dict(
            hmac_key=session_auth['hmac_key'],
@@ -1333,7 +1332,7 @@ class AuthJWT(object):
        return payload

    def refresh_token(self, orig_payload):
-        now = time.mktime(datetime.datetime.now().timetuple())
+        now = time.mktime(datetime.datetime.utcnow().timetuple())
        if self.verify_expiration:
            orig_exp = orig_payload['exp']
            if orig_exp + self.leeway < now:
@@ -1537,7 +1536,8 @@ class Auth(AuthAPI):
    # ## these are messages that can be customized
    default_messages = dict(AuthAPI.default_messages,
                            access_denied='Insufficient privileges',
-                            bulk_invite_body='You have been invited to join %(site)s, click %(link)s to complete the process',
+                            bulk_invite_body='You have been invited to join %(site)s, click %(link)s to complete '
+                                             'the process',
                            bulk_invite_subject='Invitation to join %(site)s',
                            delete_label='Check to delete',
                            email_sent='Email sent',
@@ -1762,7 +1762,7 @@ class Auth(AuthAPI):
            if auth.last_visit and auth.last_visit + delta > now:
                self.user = auth.user
                # this is a trick to speed up sessions to avoid many writes
-                if (now - auth.last_visit).seconds > (auth.expiration / 10):
+                if (now - auth.last_visit).seconds > (auth.expiration // 10):
                    auth.last_visit = now
            else:
                self.user = None
@@ -1792,6 +1792,7 @@ class Auth(AuthAPI):
                             servicevalidate='serviceValidate',
                             proxyvalidate='proxyValidate',
                             logout='logout'),
+            cas_create_user=True,
            extra_fields={},
            actions_disabled=[],
            controller=controller,
@@ -1840,14 +1841,15 @@ class Auth(AuthAPI):
        # ## these are messages that can be customized
        messages = self.messages = Messages(current.T)
        messages.update(Auth.default_messages)
-        messages.update(ajax_failed_authentication=DIV(H4('NOT AUTHORIZED'),
-                                                       'Please ',
-                                                       A('login',
-                                                         _href=self.settings.login_url +
-                                                         ('?_next=' + urllib_quote(current.request.env.http_web2py_component_location))
-                                                         if current.request.env.http_web2py_component_location else ''),
-                                                       ' to view this content.',
-                                                       _class='not-authorized alert alert-block'))
+        messages.update(ajax_failed_authentication=
+                        DIV(H4('NOT AUTHORIZED'),
+                            'Please ',
+                            A('login',
+                              _href=self.settings.login_url +
+                                    ('?_next=' + urllib_quote(current.request.env.http_web2py_component_location))
+                              if current.request.env.http_web2py_component_location else ''),
+                            ' to view this content.',
+                            _class='not-authorized alert alert-block'))
        messages.lock_keys = True

        # for "remember me" option
@@ -1876,7 +1878,7 @@ class Auth(AuthAPI):
        # _next variable in the request.
        if next:
            parts = next.split('/')
-            if not ':' in parts[0]:
+            if ':' not in parts[0]:
                return next
            elif len(parts) > 2 and parts[0].endswith(':') and parts[1:3] == ['', host]:
                return next
@@ -2008,8 +2010,7 @@ class Auth(AuthAPI):
                items.append({'name': T('Lost password?'),
                              'href': href('request_reset_password'),
                              'icon': 'icon-lock'})
-            if (self.settings.use_username and not
-                    'retrieve_username' in self.settings.actions_disabled):
+            if self.settings.use_username and 'retrieve_username' not in self.settings.actions_disabled:
                items.append({'name': T('Forgot username?'),
                              'href': href('retrieve_username'),
                              'icon': 'icon-edit'})
@@ -2181,9 +2182,7 @@ class Auth(AuthAPI):
            current_record.replace('_', ' ').title())
        for table in tables:
            fieldnames = table.fields()
-            if ('id' in fieldnames and
-                    'modified_on' in fieldnames and
-                    not current_record in fieldnames):
+            if 'id' in fieldnames and 'modified_on' in fieldnames and current_record not in fieldnames:
                table._enable_record_versioning(archive_db=archive_db,
                                                archive_name=archive_names,
                                                current_record=current_record,
@@ -2213,7 +2212,8 @@ class Auth(AuthAPI):
            fake_migrate = db._fake_migrate
        settings = self.settings
        settings.enable_tokens = enable_tokens
-        signature_list = super(Auth, self).define_tables(username, signature, migrate, fake_migrate)._table_signature_list
+        signature_list = \
+            super(Auth, self).define_tables(username, signature, migrate, fake_migrate)._table_signature_list

        now = current.request.now
        reference_table_user = 'reference %s' % settings.table_user_name
@@ -2285,6 +2285,7 @@ class Auth(AuthAPI):
        If the user doesn't yet exist, then they are created.
        """
        table_user = self.table_user()
+        create_user = self.settings.cas_create_user
        user = None
        checks = []
        # make a guess about who this user is
@@ -2317,6 +2318,11 @@ class Auth(AuthAPI):
                    update_keys[key] = keys[key]
            user.update_record(**update_keys)
        elif checks:
+            if create_user is False:
+                # Remove current open session a send message
+                self.logout(next=None, onlogout=None, log=None)
+                raise HTTP(403, "Forbidden. User need to be created first.")
+
            if 'first_name' not in keys and 'first_name' in table_user.fields:
                guess = keys.get('email', 'anonymous').split('@')[0]
                keys['first_name'] = keys.get('username', guess)
@@ -2360,7 +2366,7 @@ class Auth(AuthAPI):
            if callable(basic_auth_realm):
                basic_auth_realm = basic_auth_realm()
            elif isinstance(basic_auth_realm, (unicode, str)):
-                basic_realm = unicode(basic_auth_realm)
+                basic_realm = unicode(basic_auth_realm)  # Warning python 3.5 does not have method unicod
            elif basic_auth_realm is True:
                basic_realm = u'' + current.request.application
            http_401 = HTTP(401, u'Not Authorized', **{'WWW-Authenticate': u'Basic realm="' + basic_realm + '"'})
@@ -2462,7 +2468,7 @@ class Auth(AuthAPI):
                         _href=service + query_sep + "ticket=" + ticket)
            else:
                redirect(service + query_sep + "ticket=" + ticket)
-        if self.is_logged_in() and not 'renew' in request.vars:
+        if self.is_logged_in() and 'renew' not in request.vars:
            return allow_access()
        elif not self.is_logged_in() and 'gateway' in request.vars:
            redirect(session._cas_service)
@@ -2779,7 +2785,7 @@ class Auth(AuthAPI):
        # If auth.settings.auth_two_factor_enabled it will enable two factor
        # for all the app. Another way to anble two factor is that the user
        # must be part of a group that is called auth.settings.two_factor_authentication_group
-        if user and self.settings.auth_two_factor_enabled == True:
+        if user and self.settings.auth_two_factor_enabled is True:
            session.auth_two_factor_enabled = True
        elif user and self.settings.two_factor_authentication_group:
            role = self.settings.two_factor_authentication_group
@@ -2809,7 +2815,7 @@ class Auth(AuthAPI):
                # Set the way we generate the code or we send the code. For example using SMS...
                two_factor_methods = self.settings.two_factor_methods

-                if two_factor_methods == []:
+                if not two_factor_methods:
                    # TODO: Add some error checking to handle cases where email cannot be sent
                    self.settings.mailer.send(
                        to=user.email,
@@ -2832,47 +2838,49 @@ class Auth(AuthAPI):
                            hideerror=settings.hideerror):
                accepted_form = True

-                '''
+                """
                The lists is executed after form validation for each of the corresponding action.
                For example, in your model:

                In your models copy and paste:

-                #Before define tables, we add some extra field to auth_user
+                # Before define tables, we add some extra field to auth_user
                auth.settings.extra_fields['auth_user'] = [
                    Field('motp_secret', 'password', length=512, default='', label='MOTP Secret'),
                    Field('motp_pin', 'string', length=128, default='', label='MOTP PIN')]

-                OFFSET = 60 #Be sure is the same in your OTP Client
+                OFFSET = 60  # Be sure is the same in your OTP Client

-                #Set session.auth_two_factor to None. Because the code is generated by external app.
+                # Set session.auth_two_factor to None. Because the code is generated by external app.
                # This will avoid to use the default setting and send a code by email.
                def _set_two_factor(user, auth_two_factor):
                    return None

                def verify_otp(user, otp):
-                import time
-                from hashlib import md5
-                epoch_time = int(time.time())
-                time_start = int(str(epoch_time - OFFSET)[:-1])
-                time_end = int(str(epoch_time + OFFSET)[:-1])
-                for t in range(time_start - 1, time_end + 1):
-                    to_hash = str(t) + user.motp_secret + user.motp_pin
-                    hash = md5(to_hash).hexdigest()[:6]
-                    if otp == hash:
-                    return hash
+                    import time
+                    from hashlib import md5
+                    epoch_time = int(time.time())
+                    time_start = int(str(epoch_time - OFFSET)[:-1])
+                    time_end = int(str(epoch_time + OFFSET)[:-1])
+                    for t in range(time_start - 1, time_end + 1):
+                        to_hash = str(t) + user.motp_secret + user.motp_pin
+                        hash = md5(to_hash).hexdigest()[:6]
+                        if otp == hash:
+                        return hash

                auth.settings.auth_two_factor_enabled = True
                auth.messages.two_factor_comment = "Verify your OTP Client for the code."
-                auth.settings.two_factor_methods = [lambda user, auth_two_factor: _set_two_factor(user, auth_two_factor)]
+                auth.settings.two_factor_methods = [lambda user,
+                                                           auth_two_factor: _set_two_factor(user, auth_two_factor)]
                auth.settings.two_factor_onvalidation = [lambda user, otp: verify_otp(user, otp)]

-                '''
-                if self.settings.two_factor_onvalidation != []:
+                """
+                if self.settings.two_factor_onvalidation:

                    for two_factor_onvalidation in self.settings.two_factor_onvalidation:
                        try:
-                            session.auth_two_factor = two_factor_onvalidation(session.auth_two_factor_user, form.vars['authentication_code'])
+                            session.auth_two_factor = \
+                                two_factor_onvalidation(session.auth_two_factor_user, form.vars['authentication_code'])
                        except:
                            pass
                        else:
@@ -3655,6 +3663,16 @@ class Auth(AuthAPI):
        if not self.is_logged_in():
            redirect(self.settings.login_url,
                     client_side=self.settings.client_side)
+
+        # Go to external link to change the password
+        if self.settings.login_form != self:
+            cas = self.settings.login_form
+            # To prevent error if change_password_url function is not defined in alternate login
+            if hasattr(cas, 'change_password_url'):
+                next = cas.change_password_url(next)
+                if next is not None:
+                    redirect(next)
+
        db = self.db
        table_user = self.table_user()
        s = db(table_user.id == self.user.id)
@@ -3675,7 +3693,8 @@ class Auth(AuthAPI):
            requires = [requires]
        requires = list(filter(lambda t: isinstance(t, CRYPT), requires))
        if requires:
-            requires[0].min_length = 0
+            requires[0] = CRYPT(**requires[0].__dict__) # Copy the existing CRYPT attributes
+            requires[0].min_length = 0 # But do not enforce minimum length for the old password
        form = SQLFORM.factory(
            Field('old_password', 'password', requires=requires,
                  label=self.messages.old_password),
@@ -3759,9 +3778,9 @@ class Auth(AuthAPI):
            if any(f.compute for f in extra_fields):
                user = table_user[self.user.id]
                self._update_session_user(user)
+                self.update_groups()
            else:
                self.user.update(table_user._filter_fields(form.vars))
-
            session.flash = self.messages.profile_updated
            self.log_event(log, self.user)
            callback(onaccept, form)
@@ -4146,7 +4165,7 @@ class Auth(AuthAPI):
            archive_table = table._db[archive_table_name]
        new_record = {current_record: form.vars.id}
        for fieldname in archive_table.fields:
-            if not fieldname in ['id', current_record]:
+            if fieldname not in ['id', current_record]:
                if archive_current and fieldname in form.vars:
                    new_record[fieldname] = form.vars[fieldname]
                elif form.record and fieldname in form.record:
@@ -4956,7 +4975,10 @@ class Service(object):

            Then call it with:

-                wget --post-data '{"jsonrpc": "2.0", "id": 1, "method": "myfunction", "params": {"a": 1, "b": 2}}' http://..../app/default/call/jsonrpc2
+                wget --post-data '{"jsonrpc": "2.0",
+                                   "id": 1,
+                                   "method": "myfunction",
+                                   "params": {"a": 1, "b": 2}}' http://..../app/default/call/jsonrpc2

        """
        self.jsonrpc2_procedures[f.__name__] = f
@@ -5527,15 +5549,15 @@ def prettydate(d, T=lambda x: x, utc=False):
    else:
        suffix = ' ago'
    if dt.days >= 2 * 365:
-        return T('%d years' + suffix) % int(dt.days / 365)
+        return T('%d years' + suffix) % int(dt.days // 365)
    elif dt.days >= 365:
        return T('1 year' + suffix)
    elif dt.days >= 60:
-        return T('%d months' + suffix) % int(dt.days / 30)
+        return T('%d months' + suffix) % int(dt.days // 30)
    elif dt.days >= 27:  # 4 weeks ugly
        return T('1 month' + suffix)
    elif dt.days >= 14:
-        return T('%d weeks' + suffix) % int(dt.days / 7)
+        return T('%d weeks' + suffix) % int(dt.days // 7)
    elif dt.days >= 7:
        return T('1 week' + suffix)
    elif dt.days > 1:
@@ -5543,11 +5565,11 @@ def prettydate(d, T=lambda x: x, utc=False):
    elif dt.days == 1:
        return T('1 day' + suffix)
    elif dt.seconds >= 2 * 60 * 60:
-        return T('%d hours' + suffix) % int(dt.seconds / 3600)
+        return T('%d hours' + suffix) % int(dt.seconds // 3600)
    elif dt.seconds >= 60 * 60:
        return T('1 hour' + suffix)
    elif dt.seconds >= 2 * 60:
-        return T('%d minutes' + suffix) % int(dt.seconds / 60)
+        return T('%d minutes' + suffix) % int(dt.seconds // 60)
    elif dt.seconds >= 60:
        return T('1 minute' + suffix)
    elif dt.seconds > 1:
@@ -5600,12 +5622,12 @@ class PluginManager(object):

        where the plugin is used::

-            >>> print plugins.me.param1
+            >>> print(plugins.me.param1)
            3
-            >>> print plugins.me.param2
+            >>> print(plugins.me.param2)
            6
            >>> plugins.me.param3 = 8
-            >>> print plugins.me.param3
+            >>> print(plugins.me.param3)
            8

        Here are some tests::
@@ -5613,25 +5635,25 @@ class PluginManager(object):
            >>> a=PluginManager()
            >>> a.x=6
            >>> b=PluginManager('check')
-            >>> print b.x
+            >>> print(b.x)
            6
            >>> b=PluginManager() # reset settings
-            >>> print b.x
+            >>> print(b.x)
            <Storage {}>
            >>> b.x=7
-            >>> print a.x
+            >>> print(a.x)
            7
            >>> a.y.z=8
-            >>> print b.y.z
+            >>> print(b.y.z)
            8
            >>> test_thread_separation()
            5
            >>> plugins=PluginManager('me',db='mydb')
-            >>> print plugins.me.db
+            >>> print(plugins.me.db)
            mydb
-            >>> print 'me' in plugins
+            >>> print('me' in plugins)
            True
-            >>> print plugins.me.installed
+            >>> print(plugins.me.installed)
            True

    """
@@ -5780,7 +5802,7 @@ class Expose(object):
            return os.path.realpath(f)

    def issymlink_out(self, f):
-        "True if f is a symlink and is pointing outside of self.base"
+        """True if f is a symlink and is pointing outside of self.base"""
        return os.path.islink(f) and not self.in_base(f)

    @staticmethod
@@ -156,12 +156,12 @@ def get_digest(value):
        raise ValueError("Invalid digest algorithm: %s" % value)

 DIGEST_ALG_BY_SIZE = {
-    128 / 4: 'md5',
-    160 / 4: 'sha1',
-    224 / 4: 'sha224',
-    256 / 4: 'sha256',
-    384 / 4: 'sha384',
-    512 / 4: 'sha512',
+    128 // 4: 'md5',
+    160 // 4: 'sha1',
+    224 // 4: 'sha224',
+    256 // 4: 'sha256',
+    384 // 4: 'sha384',
+    512 // 4: 'sha512',
 }


@@ -21,7 +21,8 @@ import struct
 import decimal
 import unicodedata

-from gluon._compat import StringIO, long, basestring, unicodeT, to_unicode, urllib_unquote, unichr, to_bytes, PY2, to_unicode, to_native, string_types, urlparse
+from gluon._compat import StringIO, long, basestring, unicodeT, to_unicode, urllib_unquote, unichr, to_bytes, PY2, \
+    to_unicode, to_native, string_types, urlparse
 from gluon.utils import simple_hash, web2py_uuid, DIGEST_ALG_BY_SIZE
 from pydal.objects import Field, FieldVirtual, FieldMethod
 from functools import reduce
@@ -452,10 +453,10 @@ class IS_IN_SET(Validator):
        if not self.labels:
            items = [(k, k) for (i, k) in enumerate(self.theset)]
        else:
-            items = [(k, self.labels[i]) for (i, k) in enumerate(self.theset)]
+            items = [(k, list(self.labels)[i]) for (i, k) in enumerate(self.theset)]
        if self.sort:
            items.sort(key=lambda o: str(o[1]).upper())
-        if zero and not self.zero is None and not self.multiple:
+        if zero and self.zero is not None and not self.multiple:
            items.insert(0, ('', self.zero))
        return items

@@ -823,7 +824,7 @@ class IS_INT_IN_RANGE(Validator):

 def str2dec(number):
    s = str(number)
-    if not '.' in s:
+    if '.' not in s:
        s += '.00'
    else:
        s += '0' * (2 - len(s.split('.')[1]))
@@ -995,7 +996,7 @@ def is_empty(value, empty_regex=None):
        value = value.strip()
        if empty_regex is not None and empty_regex.match(value):
            value = ''
-    if value is None or value == '' or value == []:
+    if value is None or value == '' or value == b'' or value == []:
        return (_value, True)
    return (_value, False)

@@ -1213,7 +1214,7 @@ class IS_EMAIL(Validator):
                domain_encoded = to_unicode(domain).encode('idna').decode('ascii')
                match_domain = self.domain_regex.match(domain_encoded)

-            match = (match_body != None) and (match_domain != None)
+            match = (match_body is not None) and (match_domain is not None)
        except (TypeError, UnicodeError):
            # Value may not be a string where we can look for matches.
            # Example: we're calling ANY_OF formatter and IS_EMAIL is asked to validate a date.
@@ -1247,7 +1248,7 @@ class IS_LIST_OF_EMAILS(object):
        f = IS_EMAIL()
        for email in self.split_emails.findall(value):
            error = f(email)[1]
-            if error and not email in bad_emails:
+            if error and email not in bad_emails:
                bad_emails.append(email)
        if not bad_emails:
            return (value, None)
@@ -1461,9 +1462,9 @@ def unicode_to_ascii_authority(authority):
        if label:
            asciiLabels.append(to_native(encodings.idna.ToASCII(label)))
        else:
-             # encodings.idna.ToASCII does not accept an empty string, but
-             # it is necessary for us to allow for empty labels so that we
-             # don't modify the URL
+            # encodings.idna.ToASCII does not accept an empty string, but
+            # it is necessary for us to allow for empty labels so that we
+            # don't modify the URL
            asciiLabels.append('')
    # RFC 3490, Section 4, Step 5
    return str(reduce(lambda x, y: x + unichr(0x002E) + y, asciiLabels))
@@ -1527,13 +1528,17 @@ def unicode_to_ascii_url(url, prepend_scheme):
    if prepended:
        scheme = ''

-    unparsed = urlparse.urlunparse((scheme, unicode_to_ascii_authority(authority), escape_unicode(path), '', escape_unicode(query), str(fragment)))
+    unparsed = urlparse.urlunparse((scheme,
+                                    unicode_to_ascii_authority(authority),
+                                    escape_unicode(path),
+                                    '',
+                                    escape_unicode(query),
+                                    str(fragment)))
    if unparsed.startswith('//'):
-        unparsed = unparsed[2:] # Remove the // urlunparse puts in the beginning
+        unparsed = unparsed[2:]  # Remove the // urlunparse puts in the beginning
    return unparsed


-
 class IS_GENERIC_URL(Validator):
    """
    Rejects a URL string if any of the following is true:
@@ -2622,7 +2627,7 @@ class ANY_OF(Validator):
    def __call__(self, value):
        for validator in self.subs:
            value, error = validator(value)
-            if error == None:
+            if error is None:
                break
        return value, error

@@ -2762,7 +2767,7 @@ class LazyCrypt(object):
        else:
            digest_alg, key = self.crypt.digest_alg, ''
        if self.crypt.salt:
-            if self.crypt.salt == True:
+            if self.crypt.salt is True:
                salt = str(web2py_uuid()).replace('-', '')[-16:]
            else:
                salt = self.crypt.salt
@@ -2847,7 +2852,7 @@ class CRYPT(object):
    Supports standard algorithms

        >>> for alg in ('md5','sha1','sha256','sha384','sha512'):
-        ...     print str(CRYPT(digest_alg=alg,salt=True)('test')[0])
+        ...     print(str(CRYPT(digest_alg=alg,salt=True)('test')[0]))
        md5$...$...
        sha1$...$...
        sha256$...$...
@@ -2859,13 +2864,13 @@ class CRYPT(object):
    Supports for pbkdf2

        >>> alg = 'pbkdf2(1000,20,sha512)'
-        >>> print str(CRYPT(digest_alg=alg,salt=True)('test')[0])
+        >>> print(str(CRYPT(digest_alg=alg,salt=True)('test')[0]))
        pbkdf2(1000,20,sha512)$...$...

    An optional hmac_key can be specified and it is used as salt prefix

        >>> a = str(CRYPT(digest_alg='md5',key='mykey',salt=True)('test')[0])
-        >>> print a
+        >>> print(a)
        md5$...$...

    Even if the algorithm changes the hash can still be validated
@@ -140,7 +140,7 @@ class web2pyDialog(object):
        else:
            import tkinter
            from tkinter import messagebox
-        
+

        bg_color = 'white'
        root.withdraw()
@@ -463,7 +463,7 @@ class web2pyDialog(object):
            import tkMessageBox as messagebox
        else:
            from tkinter import messagebox
-        
+
        messagebox.showerror('web2py start server', message)

    def start(self):
@@ -1076,7 +1076,10 @@ def start_schedulers(options):
        return

    # Work around OS X problem: http://bugs.python.org/issue9405
-    import urllib
+    if PY2:
+        import urllib
+    else:
+        import urllib.request as urllib
    urllib.getproxies()

    for app in apps:
@@ -0,0 +1,38 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Written by Vinyl Darkscratch, www.queengoob.org
+
+import sys
+import os
+
+# TODO: add comments
+
+# This script can be run with no arguments (which sets the language folder to the current working directory, and default language to English), one argument (which sets the default language), or two arguments (language folder path and default language).
+# When run, this script will compare the original language's strings to their assigned values and determine if they match, just like web2py's interface.  While some phrases may be the exact same between both languages, this should provide a good idea of how far along translations are.
+
+def check_lang_progress(cwd, default_lang):
+	for x in os.listdir(cwd):
+		if x == default_lang or x.startswith("plural-"): continue
+		
+		data = eval(open(os.path.join(cwd, x)).read())
+
+		total = 0
+		translated = 0
+
+		for key in data:
+			total += 1
+			if key.replace('@markmin\x01', '') != data[key]: translated += 1
+
+		print "Translations for %s (%s): %d/%d Translated (%d Untranslated)" %(data['!langname!'], data['!langcode!'], translated, total, total-translated)
+
+if __name__ == "__main__":
+	cwd = os.getcwd()
+	default_lang = 'en'
+
+	if len(sys.argv) > 2:
+		cwd = sys.argv[1]
+		default_lang = sys.argv[2]
+	elif len(sys.argv) > 1:
+		default_lang = sys.argv[1]
+
+	check_lang_progress(cwd, default_lang)
@@ -222,7 +222,7 @@ echo <<EOF
 you can stop uwsgi and nginx with

  sudo /etc/init.d/nginx stop
-  sudo systemctl start emperor.uwsgi.service
+  sudo systemctl stop emperor.uwsgi.service
 
 and start it with

@@ -0,0 +1,82 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Written by Vinyl Darkscratch, www.queengoob.org
+
+# TODO: add comments
+
+import os
+import ast
+import sys
+import inspect
+
+currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
+parentdir = os.path.dirname(currentdir)
+sys.path.insert(0, parentdir)
+
+from gluon.cfs import getcfs
+from gluon.utf8 import Utf8
+from gluon._compat import copyreg, PY2, maketrans, iterkeys, unicodeT, to_unicode, to_bytes, iteritems, to_native, pjoin
+from gluon.languages import findT
+
+# This script can be run with no arguments (which sets the language folder to the current working directory, and default language to English), one argument (which sets the default language), or two arguments (language folder path and default language).
+# When run, it will update the default language, as well as strip all of the strings found in the non-default languages but not in the default language, and add the strings found in the default language to the non-default languages it is not, making sure translators don't do additional work that will never be used.
+
+def read_dict_aux(filename):
+	lang_text = open(filename, 'r').read().replace(b'\r\n', b'\n')
+	try:
+		return safe_eval(to_native(lang_text)) or {}
+	except Exception:
+		e = sys.exc_info()[1]
+		status = 'Syntax error in %s (%s)' % (filename, e)
+		return {'__corrupted__': status}
+
+def read_dict(filename):
+	return getcfs('lang:' + filename, filename, lambda: read_dict_aux(filename))
+
+def safe_eval(text):
+	if text.strip():
+		try:
+			return ast.literal_eval(text)
+		except ImportError:
+			return eval(text, {}, {})
+	return None
+
+def sort_function(x, y):
+	return cmp(unicode(x, 'utf-8').lower(), unicode(y, 'utf-8').lower())
+
+def write_file(file, contents):
+	file.write('# -*- coding: utf-8 -*-\n{\n')
+	for key in sorted(contents, sort_function):
+		file.write('%s: %s,\n' % (repr(Utf8(key)),
+								repr(Utf8(contents[key]))))
+	file.write('}\n')
+	file.close()
+
+def update_languages(cwd, default_lang):
+	defaultfp = os.path.join(cwd, '%s.py' %default_lang)
+	findT(cwd, default_lang)
+	default = read_dict(defaultfp)
+
+	for x in os.listdir(cwd):
+		if x == default_lang or x.startswith("plural-"): continue
+		
+		i18n = read_dict(os.path.join(cwd, x))
+		if i18n:
+			nd = default
+			for k_d1 in i18n:
+				if k_d1 in default:
+					nd[k_d1] = i18n[k_d1]
+			write_file(open(x, 'w'), nd)
+			print x
+
+if __name__ == "__main__":
+	cwd = os.getcwd()
+	default_lang = 'en'
+
+	if len(sys.argv) > 2:
+		cwd = sys.argv[1]
+		default_lang = sys.argv[2]
+	elif len(sys.argv) > 1:
+		default_lang = sys.argv[1]
+
+	update_languages(cwd, default_lang)
Author	SHA1	Message	Date
mdipierro	c9fd0fd71e	fixed test because of layout change	2017-11-14 00:55:59 -06:00
mdipierro	d59e7f35ed	fixed toolbar/configuration issue	2017-11-14 00:50:44 -06:00
mdipierro	999e7b7512	fixed layout in lack of configuration	2017-11-14 00:47:40 -06:00
mdipierro	7035398681	2.16.1	2017-11-13 23:50:20 -06:00
mdipierro	2c2c9d3aa2	moved to bootstrap4	2017-11-13 23:48:54 -06:00
mdipierro	65b3e6dda9	support for bootstrap4	2017-11-13 23:29:34 -06:00
mdipierro	d424e5d317	Merge pull request #1610 from geomapdev/dal-enhancements handle reference fields for keyed tables	2017-11-13 21:24:07 -06:00
mdipierro	a1417df67c	Merge branch 'master' of github.com:web2py/web2py	2017-11-13 21:15:04 -06:00
mdipierro	551c19bcaf	Merge pull request #1761 from josedesoto/issue/update_role_on_update_profile Update groups on edit profile	2017-11-13 21:14:02 -06:00
mdipierro	8aa07760d2	Merge pull request #1803 from leonelcamara/patch-10 Fix #1715	2017-11-13 21:13:06 -06:00
mdipierro	8bd8db5edc	Merge pull request #1804 from leonelcamara/patch-11 Update supported python versions	2017-11-13 21:12:36 -06:00
mdipierro	7c1fb6643e	fllowing pydal v17.11	2017-11-13 21:12:06 -06:00
Leonel Câmara	e44a12eaf8	Update supported python versions	2017-11-13 22:40:37 +00:00
Leonel Câmara	12253ab757	Fix #1715	2017-11-13 15:31:05 +00:00
mdipierro	30b3d84f24	fixed options when type is list	2017-11-10 22:31:35 -06:00
mdipierro	166e268308	improved handling of regex and options field attributes	2017-11-10 18:50:52 -06:00
mdipierro	14e58276cf	form handling of f.options, f.regex, f.listable, f.rearchable	2017-11-10 18:41:41 -06:00
mdipierro	eb7222aa92	Merge pull request #1802 from leonelcamara/fixsqltable Fix a problem with Expressions in SQLTABLE	2017-11-10 18:12:50 -06:00
mdipierro	009a0b87f2	Merge pull request #1801 from leonelcamara/i1800 Fixes #1800	2017-11-10 18:12:16 -06:00
mdipierro	aeb0244b2c	Merge pull request #1799 from vinyldarkscratch/master Add two scripts	2017-11-10 18:12:03 -06:00
mdipierro	57522ddeb1	Merge pull request #1797 from BigManWalter/master Set entry animation to 'fadeIn' and provide an easy switch for 'slideDown'	2017-11-10 18:10:13 -06:00
mdipierro	934042fc04	Merge pull request #1796 from vitkarpenko/middleware_status_code gluon.http expects status code as integer while LazyWSGI generates str	2017-11-10 18:09:37 -06:00
Leonel Câmara	4bfa9c1686	Fix a problem with Expressions in SQLTABLE possibly fixes #1735 and fixes web2py/pydal#487	2017-11-08 22:21:51 +00:00
Leonel Câmara	925f928843	Copy all CRYPT attributes thanks @abastardi	2017-11-08 11:53:29 +00:00
mdipierro	2f5eb409b6	show computed fields in readonly mode in appadmin	2017-11-08 00:40:47 -06:00
Leonel Câmara	228d3c41b6	Fixes #1800	2017-11-07 23:34:35 +00:00
Vinyl Darkscratch	3ab91b9fe9	Add check_lang_progress script	2017-11-06 01:42:25 -08:00
Vinyl Darkscratch	5db3b21437	Add update_languages script	2017-11-06 01:42:14 -08:00
Eric Waldman	2aa8fd22a2	Set entry animation to 'fadeIn' and provide an easy switch for 'slideDown'	2017-11-03 15:17:18 -04:00
Vitaly Karpenko	d5eb8d8f17	gluon.http expects status code as integer	2017-11-02 13:52:25 +03:00
mdipierro	c4e08eeeb3	Merge pull request #1793 from sherdim/master A step to PY3 closes #1792 and may be closes #1727	2017-11-01 18:59:52 -05:00
geomapdev	c1a3d5d67e	sync with master branch	2017-10-31 14:47:07 -07:00
geomapdev	b2841de6f3	Merge remote-tracking branch 'upstream/master'	2017-10-31 14:42:19 -07:00
Дим Щ	c7d415fefd	compatible unicode	2017-10-28 19:32:48 +03:00
Дим Щ	2e572aee9a	Update custom_import.py	2017-10-28 19:29:55 +03:00
Дим Щ	dd14d1c8c9	Add reload	2017-10-28 19:28:51 +03:00
Дим Щ	afa5fac074	Add reload	2017-10-28 19:26:34 +03:00
mdipierro	a5b76f1dec	Merge branch 'master' of github.com:web2py/web2py	2017-10-18 22:14:27 -05:00
mdipierro	1e586b32a4	Merge pull request #1789 from timnyborg/master Fix setting 'ac' in request.vars	2017-10-18 22:13:16 -05:00
mdipierro	dfcffe1a9d	Merge pull request #1785 from tiagobar/master Enhances/fixes validators/is_empty() for Python 3 compatibility, fix #1764	2017-10-18 22:12:09 -05:00
Tim Nyborg	2e6f529dfd	Fix setting 'ac' in request.vars	2017-10-12 17:18:06 +01:00
Tim Nyborg	0ae3ee506f	Merge pull request #1 from web2py/master rebase	2017-10-12 17:16:56 +01:00
geomapdev	6b103f7e3a	updated test_validators IS_IN_DB	2017-10-10 08:42:17 -07:00
geomapdev	02c32ebace	update gluon unittest for dal reference fields	2017-10-09 11:22:48 -07:00
geomapdev	603cc7092a	dal.py more code cleanup	2017-10-04 10:39:42 -07:00
geomapdev	8590aae2e8	dal.py code cleanup	2017-10-04 10:34:48 -07:00
tiago.bar	511245a68c	Unit test for #1764 fix	2017-09-23 20:21:14 -03:00
tiago.bar	4a347a4f78	Enhances/fixes validators/is_empty() for Python 3 compatibility, fix #1764	2017-09-22 13:04:47 -03:00
Jose de Soto	5f4c47729b	Removed a tab and replaced by spaces	2017-09-21 10:17:17 +02:00
mdipierro	5975e4767f	Merge pull request #1766 from abastardi/tsv_export Fix grid TSV export bug	2017-09-21 00:23:17 -05:00
mdipierro	1800304ade	Merge pull request #1763 from abastardi/compiled_views Fix bug with compiled views in compiled-only apps	2017-09-21 00:22:53 -05:00
mdipierro	9af0d8c3c9	Merge pull request #1758 from leonelcamara/patch-9 Fix #1757	2017-09-21 00:21:10 -05:00
mdipierro	2e2d3482e4	Merge pull request #1756 from leonelcamara/i1752 Make _compat accept ducks	2017-09-21 00:20:54 -05:00
mdipierro	48cc63741e	Merge pull request #1755 from leonelcamara/i1753 Fixes #1753	2017-09-21 00:20:26 -05:00
abastardi	a605e43fa6	Fix grid TSV export bug	2017-09-14 18:49:31 -04:00
abastardi	8eda21ca86	Fix bug with compiled views in compiled-only apps Compiled views were not being executed in apps containing only compiled files (i.e., generated via the admin "Pack compiled" functionality), resulting in "Invalid view" errors for all pages.	2017-09-13 11:48:25 -04:00
Jose de Soto	e8cf50326d	When profile is updated self._update_session_user(user) set session.user_groups to None. self.update_groups() needs to be done.	2017-09-13 11:21:03 +02:00
mdipierro	27c9250efb	fixed book link	2017-09-07 12:46:36 -05:00
Leonel Câmara	3ecdd1c11b	Fix #1757	2017-09-07 10:18:41 +01:00
Leonel Câmara	912c22d593	Don't use gluon.utf8.Utf8 with py3 there's no need for it and it breaks stuff	2017-09-06 16:22:50 +01:00
Leonel Câmara	4b38186b51	simplify condition if you ain't got decode you ain't bytes	2017-09-06 15:37:51 +01:00
Leonel Câmara	83f9016528	fix in py3 too	2017-09-06 15:33:47 +01:00
Leonel Câmara	a6044068cd	Fixes #1751	2017-09-06 15:31:30 +01:00
Leonel Câmara	4aefb93ab4	Fixes #1752	2017-09-06 15:28:39 +01:00
Leonel Câmara	2861dc4215	Fixes #1753	2017-09-05 15:46:31 +01:00
mdipierro	087280ec17	fixed Makefile _ssd	2017-09-01 22:47:54 -05:00
mdipierro	d06a1f9dc6	R-2.15.4	2017-09-01 22:40:20 -05:00
mdipierro	c06fc67064	following pydal 17.08	2017-08-31 17:31:43 -05:00
mdipierro	bd167aa94a	Merge pull request #1749 from liuyigh/patch-1 Minor error in EOF instructions	2017-08-31 17:29:56 -05:00
Yi Liu	c9a71a7055	MInor error in EOF instructions start -> stop	2017-08-29 11:26:22 -07:00
mdipierro	02e50cadbc	Merge pull request #1746 from abastardi/disable-submit Fix submit button disable bug	2017-08-28 11:25:55 -05:00
mdipierro	9f69ab9753	Merge pull request #1743 from ilvalle/fix_ticket_compile_views Fix view file name in ticket traceback	2017-08-28 11:25:40 -05:00
mdipierro	56d10a40c6	Merge pull request #1741 from leonelcamara/patch-8 execfile shim for python 3 compatibility	2017-08-28 11:25:28 -05:00
mdipierro	76b09393e4	Merge pull request #1738 from leonelcamara/patch-7 Fixes #1737	2017-08-28 11:25:07 -05:00
abastardi	e880da0d0e	Fix submit button disable bug When a form includes more than one submit button, after being disabled all buttons are re-enabled with the value of the first button rather than having their original values restored. This change iterates over each button separately to disable and enable.	2017-08-23 11:43:40 -04:00
ilvalle	9694c66703	Fix view file name in ticket traceback, fix #1740 , fix #1676	2017-08-19 10:07:52 +02:00
Leonel Câmara	f22e3a7624	execfile shim for python 3 compatibility Fixes #1739	2017-08-16 16:35:46 +01:00
Leonel Câmara	bd7ee209ea	Fixes #1737 getproxies is in urllib.request for python 3	2017-08-15 15:34:35 +01:00
mdipierro	5e6c3dba81	Merge pull request #1732 from lkrn/master fixes #1724	2017-08-13 13:55:09 -05:00
LAdm	135f41041d	fixes #1724 call func instead of module in url_is_acceptable	2017-08-10 06:49:36 +02:00
mdipierro	1d0f322d09	Merge pull request #1731 from leonelcamara/fix-include-files Fix include files	2017-08-09 15:46:51 -05:00
Leonel Câmara	892fba9e54	fix unordered include_files result Now only concats adjacent internal to the application stuff, this also Fixes #1673	2017-08-09 01:08:28 +01:00
Leonel Câmara	6e5c8b62cc	Make test_include_files demand the same order	2017-08-09 01:05:37 +01:00
mdipierro	852a9e0127	Merge pull request #1726 from BuhtigithuB/feature/has-membership-cached-true auth.has_membership(cached=True) check membership in auth.user_groups	2017-08-08 12:19:26 -05:00
Richard Vézina	485f868cd1	auth.has_membership(cached=True) check membership in auth.user_groups	2017-08-08 11:40:07 -04:00
mdipierro	de8b2a477b	Merge pull request #1722 from leonelcamara/patch-5 mobilize is back	2017-08-07 23:29:06 -05:00
Leonel Câmara	8533fa0d00	put is_mobile and is_tablet in the result of user_agent()	2017-08-08 00:50:55 +01:00
mdipierro	7e96ecafd7	R-2.15.3	2017-08-07 07:41:11 -05:00
mdipierro	86ea728f4f	R-2.15.3	2017-08-07 07:32:58 -05:00
mdipierro	a29947f298	websocket_messaging in Python 3 #1696 , thanks hirolau	2017-08-07 07:27:28 -05:00
mdipierro	dc29f33365	addressed __ssl.pyd issue #1716 , thanks Jhleite	2017-08-07 07:18:42 -05:00
mdipierro	834460f0cc	Merge pull request #1723 from leonelcamara/patch-6 Small unicode compatibility py3 fixes	2017-08-07 07:09:30 -05:00
mdipierro	5353e17c66	Merge pull request #1720 from leonelcamara/patch-4 Fix BEAUTIFY trying to display uploaded file contents	2017-08-07 07:08:58 -05:00
mdipierro	d2022ca500	Merge pull request #1719 from leonelcamara/patch-3 Fix response.download with nonasccii filenames	2017-08-07 07:08:33 -05:00
mdipierro	c560f607af	Merge pull request #1710 from willimoa/issue/1704 Fix Issue 1709	2017-08-07 07:08:10 -05:00
Leonel Câmara	3eea1f68f4	Make sure you return bytes when you str(body)	2017-08-07 00:20:29 +01:00
Leonel Câmara	10b6b93cb2	minor py3 compatibility change	2017-08-07 00:17:43 +01:00
Leonel Câmara	0b41ed36f9	mobilize is back Fixes #1721	2017-08-06 19:20:01 +01:00
Leonel Câmara	90e20a4f39	Fix BEAUTIFY trying to display uploaded file contents Fixes #1717	2017-08-06 17:01:27 +01:00
Leonel Câmara	a744835f21	Fix response.download with nonasccii filenames Fixes #1718	2017-08-05 13:27:44 +01:00
mdipierro	ebc614bf91	fixed has_ssl, thanks leonel	2017-08-04 10:21:43 -05:00
mdipierro	b7270df9c3	fixed issue topic/web2py/UnN6AyOh2Lg thanks Paul	2017-08-04 09:58:05 -05:00
Andrew Willimott	4a47bb8e83	d3_graph_model fixed, no longer hardcoded for a “db” database	2017-08-02 21:47:26 +12:00
mdipierro	213c4ee7d1	fixed use of whitespaces	2017-08-01 10:26:33 -05:00
mdipierro	7088b74d42	Merge pull request #1705 from josedesoto/enhancement/1557 Enhancement/1557	2017-08-01 09:46:55 -05:00
mdipierro	752b5a8cdb	Merge pull request #1704 from josedesoto/enhancement/response_json_sort_paramenter Allow response.json to be sortable or not	2017-08-01 09:46:11 -05:00
mdipierro	56e6d682d6	Merge pull request #1702 from leonelcamara/i1699 Fix autocomplete called after user_signature chk	2017-08-01 09:46:00 -05:00
mdipierro	c1881fa205	Merge pull request #1701 from jkotyz/issue/1700 Fixes 1700 - utc and local time inconsistency in AuthJWT	2017-08-01 09:45:46 -05:00
mdipierro	cc2cae5de4	Merge pull request #1692 from timnyborg/master Fix #1691: Passing tables to SQLFORM.factory() fails	2017-08-01 09:45:21 -05:00
mdipierro	cedd74c1ad	Merge pull request #1690 from josedesoto/issue/1689 del_membership prevent update self user if user_id has a value	2017-08-01 09:44:41 -05:00
Jose de Soto	d5167f2ed6	change_password_url parameter for alternate login methods	2017-07-31 19:00:24 +02:00
Jose de Soto	1014d3e86e	new parameter to auto create or not users with alternate login methods	2017-07-31 18:33:15 +02:00
Jose de Soto	f3bba29287	Allow response.json to be sortable or not	2017-07-31 17:46:46 +02:00
mdipierro	cd2ec98a26	grid.dbset	2017-07-29 01:31:25 -05:00
mdipierro	0e613f2d7f	allow DAL(..., adapter_args=dict(migrator=InDBMigrator))	2017-07-28 16:05:35 -05:00
Leonel Câmara	561828fa60	Fix autocomplete called after user_signature chk Fixes #1699 which was caused by the autocomplete widget only being called after a user_signature check and the signature not being there. The default values for user_signature and hash_vars make this work with the grid when it has the user_signature=True option without any problem when that isn't the case. Users can disable it if they want or change it according to what they're verifying in their controllers. The autocomplete widget will still fail with the default kwargs in a situation where it is called in a controller function that first verifies the URL signature with hash_vars=True. For users having that problem, there's no way around this without inserting a security flaw, so they must sadly change their application code to give the autocomplete widget the needed user_signature and hash_vars argument.	2017-07-28 02:20:54 +01:00
Jan Kotyz	19efbfecfa	Fixes 1700	2017-07-27 11:27:41 +02:00
Tim Nyborg	d43604c3ff	List comprehension rather than loop I've now learned this is quicker	2017-07-24 10:15:12 +01:00
Tim Nyborg	ec21f72ce3	Fixes #1691 Clone fields, but leave tables untouched in factory()	2017-07-21 10:20:08 +01:00
Jose de Soto	aa0313c59b	del_membership prevent update self user if user_id has a value	2017-07-21 10:41:58 +02:00
mdipierro	0a013e3edc	R-2.15.2	2017-07-19 01:21:48 -05:00
mdipierro	fb4c114d85	Merge pull request #1686 from leonelcamara/25issues Fixes 6 issues	2017-07-18 03:44:19 -05:00
mdipierro	b47e1334d5	Merge pull request #1682 from leonelcamara/fixheroku Fixheroku	2017-07-18 03:43:45 -05:00
Leonel Câmara	ce0b255747	Fixes #1672	2017-07-16 17:59:52 +01:00
Leonel Câmara	05d2ced779	remove print that was left from debugging	2017-07-16 14:27:11 +01:00
Leonel Câmara	d144ff7d65	Fixes #1687	2017-07-16 14:06:03 +01:00
Leonel Câmara	780510dc32	Fixes #1684	2017-07-15 09:52:58 +01:00
Leonel Câmara	7a5f611e76	Fixes #1571	2017-07-14 20:23:13 +01:00
Leonel Câmara	b7b8a009f2	Fixes #1680	2017-07-14 20:17:30 +01:00
Leonel Câmara	113df51ef9	Fixes #1685	2017-07-14 18:52:36 +01:00
mdipierro	4e704ca6f7	R-2.15.1	2017-07-10 16:18:35 -05:00
mdipierro	047ed786ac	fixed some error in hashlib_md5	2017-07-10 16:02:15 -05:00
mdipierro	ca1e5156ba	fixed hashlib_md5 again, thanks Paolo	2017-07-10 15:10:16 -05:00
mdipierro	4854b84ff9	fixed a python 3 problem, thanks kato	2017-07-10 14:54:08 -05:00
mdipierro	aa252cdbd8	Merge branch 'BuhtigithuB-improve/pep8-globals-py'	2017-07-10 14:20:45 -05:00
mdipierro	e3ec4d4075	merged hashlib conflict	2017-07-10 14:20:39 -05:00
mdipierro	81b000d47a	Merge branch 'master' of github.com:web2py/web2py	2017-07-10 14:19:08 -05:00
mdipierro	305dac4976	fixed changelog	2017-07-10 14:19:01 -05:00
mdipierro	876cc634a8	Merge pull request #1679 from leonelcamara/fix1675 Fixes #1675 dropdown not working in the "manage" button	2017-07-10 14:18:17 -05:00
mdipierro	c004d2c16e	Merge pull request #1678 from willimoa/issue/1674 Issue/1674	2017-07-10 14:17:56 -05:00
mdipierro	e3cce4d752	Merge pull request #1669 from ilvalle/testing_pypy3_trusty Testing pypy3.5 in travis, switching to ubuntu trusty	2017-07-10 14:17:20 -05:00
mdipierro	2c84b88466	Merge pull request #1668 from ilvalle/fix_compile_views fix performance regression with compiled views	2017-07-10 14:17:04 -05:00
mdipierro	fe652c851b	Merge pull request #1667 from BuhtigithuB/improve/pep8-main-py Enhance main.py PEP8	2017-07-10 14:16:09 -05:00
mdipierro	c183c7b18b	Merge pull request #1666 from abastardi/ajax-uploads Enable Ajax file uploads	2017-07-10 14:15:01 -05:00
mdipierro	e2c9875cd5	Merge pull request #1665 from BuhtigithuB/clean/python-2-6-remaining-code-in-import-all-py Close #1664 remaining py2.6 specific code	2017-07-10 14:14:44 -05:00
mdipierro	579c54f926	Merge pull request #1663 from BuhtigithuB/improve/pep8-http-py Enhance http.py PEP8	2017-07-10 14:14:09 -05:00
mdipierro	b2cb0bc189	Merge pull request #1662 from BuhtigithuB/improve/pep8-html-py Enhance html.py PEP8	2017-07-10 14:13:53 -05:00
mdipierro	3f6e8c755a	Merge pull request #1660 from BuhtigithuB/improve/pep8-highlight-py Enhance highlight.py PEP8	2017-07-10 14:13:19 -05:00
mdipierro	023d7f3e6c	Merge pull request #1659 from BuhtigithuB/improve/pep8-decoder-py Enhance decoder.py PEP8	2017-07-10 14:13:04 -05:00
mdipierro	01285ad4cd	Merge pull request #1657 from BuhtigithuB/improve/pep8-dal-py Enhance dal.py PEP8	2017-07-10 14:12:55 -05:00
mdipierro	0d855c1e9c	Merge pull request #1656 from BuhtigithuB/improve/pep8-contenttype-py Enhance contenttype.py PEP8	2017-07-10 14:12:45 -05:00
mdipierro	2d21c00e8d	Merge pull request #1655 from BuhtigithuB/improve/pep8-compileapp-py Enhance compileapp.py PEP8	2017-07-10 14:12:30 -05:00
mdipierro	de9d0eb895	Merge pull request #1654 from BuhtigithuB/improve/pep8-admin-py Enhance admin.py PEP8	2017-07-10 14:12:16 -05:00
mdipierro	9998916ef6	Merge pull request #1653 from BuhtigithuB/improve/pep8-validators-py Enhance validators.py PEP8 and fix docstring py3 compatibility	2017-07-10 14:11:58 -05:00
mdipierro	453123a8ed	Merge pull request #1652 from BuhtigithuB/improve/pep8-tools-py Enhance tools.py PEP8 compliancy	2017-07-10 14:11:11 -05:00
mdipierro	2396cad2d1	Merge pull request #1658 from ilvalle/fix_minify fix minify, added tests	2017-07-10 14:09:53 -05:00
mdipierro	540eecc207	R-2.15.0b2	2017-07-10 03:23:13 -05:00
mdipierro	83681f3f5d	fixed markmin test, change beahvior	2017-07-10 03:16:42 -05:00
mdipierro	b0a01ef720	fixed some relative imports, thanks steve.van.christen	2017-07-10 03:13:12 -05:00
mdipierro	da5543f62e	tracking newest pydal	2017-07-10 03:06:08 -05:00
mdipierro	dc4ff7c3cc	made markmin2html independens on gluon again and fixed reported problem, thanks mweissen	2017-07-10 03:01:35 -05:00
mdipierro	0223b0871e	jquery 2.2.4 in admin	2017-07-10 02:43:56 -05:00
Leonel Câmara	b3a7c20f3f	Fixes #1635	2017-07-10 00:17:15 +01:00
Leonel Câmara	2fc4115718	update heroku.py to the new pydal	2017-07-09 15:35:48 +01:00
Leonel Câmara	c6c027dbec	Fixes #1675 dropdown not working in the "manage" button	2017-07-07 18:10:31 +01:00
Andrew Willimott	60edb11420	Explicit path added for graph css file.	2017-07-07 21:03:55 +12:00
Andrew Willimott	51ce3ffd36	Path to css file changed to explicitly point to admin app	2017-07-07 20:39:42 +12:00
mdipierro	89832479fc	R-2.16.0b1	2017-07-05 01:48:28 -05:00
mdipierro	15ffdd9a20	R-2.16.0b1	2017-07-05 01:46:01 -05:00
mdipierro	8505c7b282	R-2.16.0b1	2017-07-05 01:42:59 -05:00
ilvalle	af7bfac1e2	Testing pypy3.5 in travis, switching to ubuntu trusty	2017-06-24 08:07:50 +02:00
ilvalle	054320820c	fix performance regression with compiled views	2017-06-24 07:21:23 +02:00
Richard Vézina	1a52b0ee3b	Enhance main.py PEP8	2017-06-22 09:46:55 -04:00
abastardi	78f3af6fc1	Use FormData when supported to allow Ajax file uploads	2017-06-21 16:29:19 -04:00
Richard Vézina	b5b98d6e19	Close #1664 remaining py2.6 specific code	2017-06-21 15:32:28 -04:00
Richard Vézina	aa1b71e431	Enhance http.py PEP8	2017-06-21 15:14:41 -04:00
Richard Vézina	472c0ff2fb	Enhance html.py PEP8	2017-06-21 15:08:52 -04:00
Richard Vézina	58bedd4c1a	Enhance highlight.py PEP8	2017-06-21 14:54:59 -04:00
Richard Vézina	88790dcaee	Enhance globals.py PEP8	2017-06-21 14:49:10 -04:00
Richard Vézina	a8fb41333b	Enhance decoder.py PEP8	2017-06-21 14:38:52 -04:00
Richard Vézina	8d5464692f	Enhance dal.py PEP8	2017-06-21 13:41:10 -04:00
Richard Vézina	2080e0460f	Enhance contenttype.py PEP8	2017-06-21 13:32:42 -04:00
Richard Vézina	7f5fc798c5	Enhance compileapp.py PEP8	2017-06-21 13:30:02 -04:00
Richard Vézina	833cb03ee1	Enhance admin.py PEP8	2017-06-21 13:22:30 -04:00
Richard Vézina	590de9c890	Enhance validators.py PEP8 and fix docstring py3 compatibility	2017-06-21 11:57:31 -04:00
Richard Vézina	583d106104	Fix docstring py3 compatibility issues print -> print()	2017-06-21 11:33:00 -04:00
Richard Vézina	7ada2cf89a	Enhance tools.py PEP8 compliancy	2017-06-21 11:27:54 -04:00
ilvalle	9f79dccb05	fix minify, added tests	2017-06-20 22:12:43 +02:00
geomapdev	69e6e79e23	Update dal.py updated to handle references without format	2017-05-01 08:39:31 -07:00
geomapdev	3292f760ca	handle reference fields for keyed tables	2017-04-20 10:21:16 -07:00