testing revert in pydal

Added row class to form-group so form becomes inline in Bootstrap v4

.travis.yml

@@ -10,12 +10,16 @@ python:
   - '2.7'
   - '3.5'
   - '3.6'
+  - '3.6-dev'
+  - '3.7-dev'
   - 'pypy-5.3.1'
   - 'pypy3.5-5.7.1-beta'
 
 matrix:
   allow_failures:
     - python: 'pypy3.5-5.7.1-beta'
+    - python: '3.6-dev'
+    - python: '3.7-dev'
 
 install:
   - pip install -e .

CHANGELOG

@@ -1,4 +1,11 @@
-## 2.15.0b1
+## 2.16.1
+- pydal 17.11
+- bootstrap 4 
+- better welcome examples
+- many bug fixes
+
+## 2.15.1-4
+- pydal 17.08
 - dropped support for python 2.6
 - dropped web shell
 - experimental python 3 support

Makefile

@@ -30,11 +30,7 @@ update:
 	wget -O gluon/contrib/feedparser.py http://feedparser.googlecode.com/svn/trunk/feedparser/feedparser.py
 	wget -O gluon/contrib/simplejsonrpc.py http://rad2py.googlecode.com/hg/ide2py/simplejsonrpc.py
 	echo "remember that pymysql was tweaked"
-src:
+rmfiles:
-	### Use semantic versioning
-	echo 'Version 2.15.2-stable+timestamp.'`date +%Y.%m.%d.%H.%M.%S` > VERSION
-	### rm -f all junk files
-	make clean
 	### clean up baisc apps
 	rm -f routes.py 
 	rm -rf applications/*/sessions/*
@@ -46,6 +42,12 @@ src:
 	rm -rf applications/admin/uploads/*                 
 	rm -rf applications/welcome/uploads/*               
 	rm -rf applications/examples/uploads/* 
+src:
+	### Use semantic versioning
+	echo 'Version 2.16.1-stable+timestamp.'`date +%Y.%m.%d.%H.%M.%S` > VERSION
+	### rm -f all junk files
+	#make clean
+	# make rmfiles
 	### make welcome layout and appadmin the default
 	cp applications/welcome/views/appadmin.html applications/admin/views
 	cp applications/welcome/views/appadmin.html applications/examples/views
@@ -54,7 +56,7 @@ src:
 	### build web2py_src.zip
 	echo '' > NEWINSTALL
 	mv web2py_src.zip web2py_src_old.zip | echo 'no old'
-	cd ..; zip -r web2py/web2py_src.zip web2py/web2py.py web2py/anyserver.py web2py/fabfile.py web2py/gluon/* web2py/extras/* web2py/handlers/* web2py/examples/* web2py/README.markdown  web2py/LICENSE web2py/CHANGELOG web2py/NEWINSTALL web2py/VERSION web2py/MANIFEST.in web2py/scripts/*.sh web2py/scripts/*.py web2py/applications/admin web2py/applications/examples/ web2py/applications/welcome web2py/applications/__init__.py web2py/site-packages/__init__.py web2py/gluon/tests/*.sh web2py/gluon/tests/*.py
+	cd ..; zip -r --exclude=**.git** web2py/web2py_src.zip web2py/web2py.py web2py/anyserver.py web2py/fabfile.py web2py/gluon/* web2py/extras/* web2py/handlers/* web2py/examples/* web2py/README.markdown  web2py/LICENSE web2py/CHANGELOG web2py/NEWINSTALL web2py/VERSION web2py/MANIFEST.in web2py/scripts/*.sh web2py/scripts/*.py web2py/applications/admin web2py/applications/examples/ web2py/applications/welcome web2py/applications/__init__.py web2py/site-packages/__init__.py web2py/gluon/tests/*.sh web2py/gluon/tests/*.py
 
 mdp:
 	make src
@@ -97,6 +99,8 @@ win:
 	cp -r applications/welcome ../web2py_win/web2py/applications
 	cp -r applications/examples ../web2py_win/web2py/applications
 	cp applications/__init__.py ../web2py_win/web2py/applications
+	# per https://github.com/web2py/web2py/issues/1716
+	mv ../web2py_win/web2py/_ssl.pyd ../web2py_win/web2py/_ssl.pyd.legacy | echo 'done'
 	cd ../web2py_win; zip -r web2py_win.zip web2py
 	mv ../web2py_win/web2py_win.zip .
 run:

VERSION

@@ -1 +1 @@
-Version 2.15.2-stable+timestamp.2017.07.19.01.21.31
+Version 2.16.1-stable+timestamp.2017.11.13.23.50.07

applications/admin/controllers/appadmin.py

@@ -657,37 +657,36 @@ def d3_graph_model():
     Create a list of table dicts, called "nodes"
     """
     
-    data = {}
     nodes = []
     links = []
 
-    subgraphs = dict()
+    for database in databases:
+        db = eval_in_global_env(database)
+        for tablename in db.tables:
+            fields = []
+            for field in db[tablename]:
+                f_type = field.type
+                if not isinstance(f_type,str):
+                    disp = ' '
+                elif f_type == 'string':
+                    disp =  field.length
+                elif f_type == 'id':
+                    disp =  "PK"
+                elif f_type.startswith('reference') or \
+                    f_type.startswith('list:reference'):
+                    disp = "FK"
+                else:
+                    disp = ' '
+                fields.append(dict(name= field.name, type=field.type, disp = disp))
 
-    for tablename in db.tables:
+                if isinstance(f_type,str) and (
-        fields = []
+                    f_type.startswith('reference') or
-        for field in db[tablename]:
+                    f_type.startswith('list:reference')):
-            f_type = field.type
+                    referenced_table = f_type.split()[1].split('.')[0]
-            if not isinstance(f_type,str):
-                disp = ' '
-            elif f_type == 'string':
-                disp =  field.length
-            elif f_type == 'id':
-                disp =  "PK"
-            elif f_type.startswith('reference') or \
-                f_type.startswith('list:reference'):
-                disp = "FK"
-            else:
-                disp = ' '
-            fields.append(dict(name= field.name, type=field.type, disp = disp))
 
-            if isinstance(f_type,str) and (
+                    links.append(dict(source=tablename, target = referenced_table))
-                f_type.startswith('reference') or
-                f_type.startswith('list:reference')):
-                referenced_table = f_type.split()[1].split('.')[0]
 
-                links.append(dict(source=tablename, target = referenced_table))
+            nodes.append(dict(name=tablename, type="table", fields = fields))
-
-        nodes.append(dict(name=tablename, type="table", fields = fields))
 
     # d3 v4 allows individual modules to be specified.  The complete d3 library is included below.
     response.files.append(URL('admin','static','js/d3.min.js'))

applications/admin/controllers/default.py

@@ -963,8 +963,7 @@ def edit_language():
         form = SPAN(strings['__corrupted__'], _class='error')
         return dict(filename=filename, form=form)
 
-    keys = sorted(strings.keys(), lambda x, y: cmp(
+    keys = sorted(strings.keys(), key=lambda x: to_native(x).lower())
-        unicode(x, 'utf-8').lower(), unicode(y, 'utf-8').lower()))
     rows = []
     rows.append(H2(T('Original/Translation')))
 

applications/admin/controllers/pythonanywhere.py

@@ -5,7 +5,7 @@ import re
 import gzip
 import tarfile
 from gluon.contrib.simplejsonrpc import ServerProxy
-from gluon._compat import StringIO, ProtocolError
+from gluon._compat import StringIO, ProtocolError, urlencode, urllib2
 
 def deploy():
     response.title = T('Deploy to pythonanywhere')
@@ -26,9 +26,8 @@ def create_account():
         except ProtocolError as error:
             pass
 
-    import urllib, urllib2
     url = 'https://www.pythonanywhere.com/api/web2py/create_account'
-    data = urllib.urlencode(request.vars)
+    data = urlencode(request.vars)
     req = urllib2.Request(url, data)
     
     try:

applications/admin/static/js/web2py.js

@@ -12,6 +12,10 @@
         $.error('web2py.js has already been loaded!');
     }
 
+    var FORMDATA_IS_SUPPORTED = typeof(FormData) !== 'undefined';
+    var animateIn = 'fadeIn';
+    // var animateIn = 'slideDown';
+
     String.prototype.reverse = function () {
         return this.split('').reverse().join('');
     };
@@ -178,7 +182,7 @@
         },
         /* manage errors in forms */
         manage_errors: function (target) {
-            $('div.error', target).hide().slideDown('slow');
+            $('div.error', target).hide()[animateIn]('slow');
         },
         after_ajax: function (xhr) {
             /* called whenever an ajax request completes */
@@ -263,13 +267,17 @@
                 }
             });
             /* help preventing double form submission for normal form (not LOADed) */
-            $(doc).on('submit', 'form', function () {
+            $(doc).on('submit', 'form', function (e) {
-                var submit_button = $(this).find(web2py.formInputClickSelector);
+                var submit_buttons = $(this).find(web2py.formInputClickSelector);
-                web2py.disableElement(submit_button);
+                submit_buttons.each(function() {
+                    web2py.disableElement($(this));
+                })
                 /* safeguard in case the form doesn't trigger a refresh,
                 see https://github.com/web2py/web2py/issues/1100 */
                 setTimeout(function () {
-                    web2py.enableElement(submit_button);
+                    submit_buttons.each(function() {
+                        web2py.enableElement($(this));
+                    });
                 }, 5000);
             });
             doc.ajaxSuccess(function (e, xhr) {
@@ -320,7 +328,15 @@
                 form.submit(function (e) {
                     web2py.disableElement(form.find(web2py.formInputClickSelector));
                     web2py.hide_flash();
-                    web2py.ajax_page('post', url, form.serialize(), target, form);
+
+                    var formData;
+                    if (FORMDATA_IS_SUPPORTED) {
+                        formData = new FormData(form[0]); // Allows file uploads.
+                    } else {
+                        formData = form.serialize(); // Fallback for older browsers.
+                    }
+                    web2py.ajax_page('post', url, formData, target, form);
+
                     e.preventDefault();
                 });
                 form.on('click', web2py.formInputClickSelector, function (e) {
@@ -339,11 +355,18 @@
             if (web2py.isUndefined(element)) element = $(document);
             /* if target is not there, fill it with something that there isn't in the page*/
             if (web2py.isUndefined(target) || target === '') target = 'w2p_none';
+
+            /* processData and contentType must be set to false when passing a FormData
+               object to jQuery.ajax. */
+            var isFormData = Object.prototype.toString.call(data) === '[object FormData]';
+            var contentType = isFormData ? false : 'application/x-www-form-urlencoded; charset=UTF-8';
             if (web2py.fire(element, 'ajax:before', null, target)) { /*test a usecase, should stop here if returns false */
                 $.ajax({
                     'type': method,
                     'url': action,
                     'data': data,
+                    'processData': !isFormData,
+                    'contentType': contentType,
                     'beforeSend': function (xhr, settings) {
                         xhr.setRequestHeader('web2py-component-location', document.location);
                         xhr.setRequestHeader('web2py-component-element', target);
@@ -594,8 +617,8 @@
         flash: function (message, status) {
             var flash = $('.w2p_flash');
             web2py.hide_flash();
-            flash.html(message).addClass(status);
+            flash.text(message).addClass(status);
-            if (flash.html()) flash.append('<span id="closeflash"> &times; </span>').slideDown();
+            if (flash.html()) flash.append('<span id="closeflash"> &times; </span>')[animateIn]();
         },
         hide_flash: function () {
             $('.w2p_flash').fadeOut(0).html('');
@@ -609,7 +632,7 @@
                 for (var k = 0; k < triggers[id].length; k++) {
                     var dep = $('#' + triggers[id][k], target);
                     var tr = $('#' + triggers[id][k] + '__row', target);
-                    if (t.is(dep.attr('data-show-if'))) tr.slideDown();
+                    if (t.is(dep.attr('data-show-if'))) tr[animateIn]();
                     else tr.hide();
                 }
             };
@@ -699,8 +722,9 @@
             });
         },
         /* Disables form elements:
+        - Does not disable elements with 'data-w2p_disable' attribute
         - Caches element value in 'w2p_enable_with' data store
-        - Replaces element text with value of 'data-disable-with' attribute
+        - Replaces element text with value of 'data-w2p_disable_with' attribute
         - Sets disabled property to true
         */
         disableFormElements: function (form) {
@@ -712,13 +736,15 @@
                 if (!web2py.isUndefined(disable)) {
                     return false;
                 }
-                if (web2py.isUndefined(disable_with)) {
+                if (!element.is(':file')) { // Altering file input values is not allowed.
-                    element.data('w2p_disable_with', element[method]());
+                    if (web2py.isUndefined(disable_with)) {
+                        element.data('w2p_disable_with', element[method]());
+                    }
+                    if (web2py.isUndefined(element.data('w2p_enable_with'))) {
+                        element.data('w2p_enable_with', element[method]());
+                    }
+                    element[method](element.data('w2p_disable_with'));
                 }
-                if (web2py.isUndefined(element.data('w2p_enable_with'))) {
-                    element.data('w2p_enable_with', element[method]());
-                }
-                element[method](element.data('w2p_disable_with'));
                 element.prop('disabled', true);
             });
         },

applications/examples/controllers/appadmin.py

@@ -657,37 +657,36 @@ def d3_graph_model():
     Create a list of table dicts, called "nodes"
     """
     
-    data = {}
     nodes = []
     links = []
 
-    subgraphs = dict()
+    for database in databases:
+        db = eval_in_global_env(database)
+        for tablename in db.tables:
+            fields = []
+            for field in db[tablename]:
+                f_type = field.type
+                if not isinstance(f_type,str):
+                    disp = ' '
+                elif f_type == 'string':
+                    disp =  field.length
+                elif f_type == 'id':
+                    disp =  "PK"
+                elif f_type.startswith('reference') or \
+                    f_type.startswith('list:reference'):
+                    disp = "FK"
+                else:
+                    disp = ' '
+                fields.append(dict(name= field.name, type=field.type, disp = disp))
 
-    for tablename in db.tables:
+                if isinstance(f_type,str) and (
-        fields = []
+                    f_type.startswith('reference') or
-        for field in db[tablename]:
+                    f_type.startswith('list:reference')):
-            f_type = field.type
+                    referenced_table = f_type.split()[1].split('.')[0]
-            if not isinstance(f_type,str):
-                disp = ' '
-            elif f_type == 'string':
-                disp =  field.length
-            elif f_type == 'id':
-                disp =  "PK"
-            elif f_type.startswith('reference') or \
-                f_type.startswith('list:reference'):
-                disp = "FK"
-            else:
-                disp = ' '
-            fields.append(dict(name= field.name, type=field.type, disp = disp))
 
-            if isinstance(f_type,str) and (
+                    links.append(dict(source=tablename, target = referenced_table))
-                f_type.startswith('reference') or
-                f_type.startswith('list:reference')):
-                referenced_table = f_type.split()[1].split('.')[0]
 
-                links.append(dict(source=tablename, target = referenced_table))
+            nodes.append(dict(name=tablename, type="table", fields = fields))
-
-        nodes.append(dict(name=tablename, type="table", fields = fields))
 
     # d3 v4 allows individual modules to be specified.  The complete d3 library is included below.
     response.files.append(URL('admin','static','js/d3.min.js'))

applications/examples/private/content/en/default/what/whyweb2py.markmin

@@ -3,9 +3,9 @@
 - **Created by a community of professionals** and University professors in Computer Science and Software Engineering.
 - **Always backward compatible.** We have not broken backward compatibility since version 1.0 in 2007, and we pledge not to break it in the future.
 - **Easy to run.** It requires no installation and no configuration.
-- **Runs on** Windows, Mac, Unix/Linux, Google App Engine, Amazon EC2, and almost any web hosting via Python 2.5/2.6/2.7/pypy, or Java with Jython.
+- **Runs on** Windows, Mac, Unix/Linux, Google App Engine, Amazon EC2, and almost any web hosting via Python 2.7/3.5/3.6/pypy.
-- **Runs with** Apache, Lighttpd, Cherokee and almost any other web server via CGI, FastCGI, WSGI, mod_proxy, and/or mod_python. It can embed third party WSGI apps and middleware.
+- **Runs with** Apache, Nginx, Lighttpd, Cherokee and almost any other web server via CGI, FastCGI, WSGI, mod_proxy, and/or mod_python. It can embed third party WSGI apps and middleware.
-- **Talks to** SQLite, PostgreSQL, MySQL, MSSQL, FireBird, Oracle, IBM DB2, Informix, Ingres, and Google App Engine. 
+- **Talks to** SQLite, PostgreSQL, MySQL, MSSQL, FireBird, Sybase, Oracle, IBM DB2, Informix, Ingres, MongoDB, and Google App Engine. 
 - **Secure** [[It prevents the most common types of vulnerabilities http://web2py.com/examples/default/security]] including Cross Site Scripting, Injection Flaws, and Malicious File Execution.
 - **Enforces good Software Engineering practices** (Model-View-Controller design, Server-side form validation, postbacks) that make the code more readable, scalable, and maintainable.
 - **Speaks multiple protocols** HTML/XML, RSS/ATOM, RTF, PDF, JSON, AJAX, XML-RPC, CSV, REST, WIKI, Flash/AMF, and Linked Data (RDF). 

applications/examples/static/js/web2py.js

@@ -12,6 +12,10 @@
         $.error('web2py.js has already been loaded!');
     }
 
+    var FORMDATA_IS_SUPPORTED = typeof(FormData) !== 'undefined';
+    var animateIn = 'fadeIn';
+    // var animateIn = 'slideDown';
+
     String.prototype.reverse = function () {
         return this.split('').reverse().join('');
     };
@@ -178,7 +182,7 @@
         },
         /* manage errors in forms */
         manage_errors: function (target) {
-            $('div.error', target).hide().slideDown('slow');
+            $('div.error', target).hide()[animateIn]('slow');
         },
         after_ajax: function (xhr) {
             /* called whenever an ajax request completes */
@@ -263,13 +267,17 @@
                 }
             });
             /* help preventing double form submission for normal form (not LOADed) */
-            $(doc).on('submit', 'form', function () {
+            $(doc).on('submit', 'form', function (e) {
-                var submit_button = $(this).find(web2py.formInputClickSelector);
+                var submit_buttons = $(this).find(web2py.formInputClickSelector);
-                web2py.disableElement(submit_button);
+                submit_buttons.each(function() {
+                    web2py.disableElement($(this));
+                })
                 /* safeguard in case the form doesn't trigger a refresh,
                 see https://github.com/web2py/web2py/issues/1100 */
                 setTimeout(function () {
-                    web2py.enableElement(submit_button);
+                    submit_buttons.each(function() {
+                        web2py.enableElement($(this));
+                    });
                 }, 5000);
             });
             doc.ajaxSuccess(function (e, xhr) {
@@ -320,7 +328,15 @@
                 form.submit(function (e) {
                     web2py.disableElement(form.find(web2py.formInputClickSelector));
                     web2py.hide_flash();
-                    web2py.ajax_page('post', url, form.serialize(), target, form);
+
+                    var formData;
+                    if (FORMDATA_IS_SUPPORTED) {
+                        formData = new FormData(form[0]); // Allows file uploads.
+                    } else {
+                        formData = form.serialize(); // Fallback for older browsers.
+                    }
+                    web2py.ajax_page('post', url, formData, target, form);
+
                     e.preventDefault();
                 });
                 form.on('click', web2py.formInputClickSelector, function (e) {
@@ -339,11 +355,18 @@
             if (web2py.isUndefined(element)) element = $(document);
             /* if target is not there, fill it with something that there isn't in the page*/
             if (web2py.isUndefined(target) || target === '') target = 'w2p_none';
+
+            /* processData and contentType must be set to false when passing a FormData
+               object to jQuery.ajax. */
+            var isFormData = Object.prototype.toString.call(data) === '[object FormData]';
+            var contentType = isFormData ? false : 'application/x-www-form-urlencoded; charset=UTF-8';
             if (web2py.fire(element, 'ajax:before', null, target)) { /*test a usecase, should stop here if returns false */
                 $.ajax({
                     'type': method,
                     'url': action,
                     'data': data,
+                    'processData': !isFormData,
+                    'contentType': contentType,
                     'beforeSend': function (xhr, settings) {
                         xhr.setRequestHeader('web2py-component-location', document.location);
                         xhr.setRequestHeader('web2py-component-element', target);
@@ -594,8 +617,8 @@
         flash: function (message, status) {
             var flash = $('.w2p_flash');
             web2py.hide_flash();
-            flash.html(message).addClass(status);
+            flash.text(message).addClass(status);
-            if (flash.html()) flash.append('<span id="closeflash"> &times; </span>').slideDown();
+            if (flash.html()) flash.append('<span id="closeflash"> &times; </span>')[animateIn]();
         },
         hide_flash: function () {
             $('.w2p_flash').fadeOut(0).html('');
@@ -609,7 +632,7 @@
                 for (var k = 0; k < triggers[id].length; k++) {
                     var dep = $('#' + triggers[id][k], target);
                     var tr = $('#' + triggers[id][k] + '__row', target);
-                    if (t.is(dep.attr('data-show-if'))) tr.slideDown();
+                    if (t.is(dep.attr('data-show-if'))) tr[animateIn]();
                     else tr.hide();
                 }
             };
@@ -699,8 +722,9 @@
             });
         },
         /* Disables form elements:
+        - Does not disable elements with 'data-w2p_disable' attribute
         - Caches element value in 'w2p_enable_with' data store
-        - Replaces element text with value of 'data-disable-with' attribute
+        - Replaces element text with value of 'data-w2p_disable_with' attribute
         - Sets disabled property to true
         */
         disableFormElements: function (form) {
@@ -712,13 +736,15 @@
                 if (!web2py.isUndefined(disable)) {
                     return false;
                 }
-                if (web2py.isUndefined(disable_with)) {
+                if (!element.is(':file')) { // Altering file input values is not allowed.
-                    element.data('w2p_disable_with', element[method]());
+                    if (web2py.isUndefined(disable_with)) {
+                        element.data('w2p_disable_with', element[method]());
+                    }
+                    if (web2py.isUndefined(element.data('w2p_enable_with'))) {
+                        element.data('w2p_enable_with', element[method]());
+                    }
+                    element[method](element.data('w2p_disable_with'));
                 }
-                if (web2py.isUndefined(element.data('w2p_enable_with'))) {
-                    element.data('w2p_enable_with', element[method]());
-                }
-                element[method](element.data('w2p_disable_with'));
                 element.prop('disabled', true);
             });
         },

applications/examples/views/default/download.html

@@ -48,7 +48,7 @@
       </tr>
       <tr>
         <td>
-          <a class="btn btn180 rounded green" href="https://dl.dropbox.com/u/18065445/web2py/web2py_manual_5th.pdf">Manual</a>
+          <a class="btn btn180 rounded green" href="http://mdipierro.github.io/web2py/web2py_manual_5th.pdf">Manual</a>
         </td>
         <td>
           <a class="btn btn180 rounded" href="https://github.com/web2py/web2py/releases">Change Log</a>

applications/welcome/controllers/appadmin.py

@@ -394,10 +394,9 @@ def ccache():
         redirect(URL(r=request))
     
     try:
-        from guppy import hpy
+        from pympler.asizeof import asizeof
-        hp = hpy()
     except ImportError:
-        hp = False
+        asizeof = False
 
     import shelve
     import os
@@ -451,9 +450,9 @@ def ccache():
             ram['ratio'] = 0
 
         for key, value in iteritems(cache.ram.storage):
-            if hp:
+            if asizeof:
-                ram['bytes'] += hp.iso(value[1]).size
+                ram['bytes'] += asizeof(value[1])
-                ram['objects'] += hp.iso(value[1]).count
+                ram['objects'] += 1
             ram['entries'] += 1
             if value[0] < ram['oldest']:
                 ram['oldest'] = value[0]
@@ -469,9 +468,9 @@ def ccache():
                 except (KeyError, ZeroDivisionError):
                     disk['ratio'] = 0
             else:
-                if hp:
+                if asizeof:
-                    disk['bytes'] += hp.iso(value[1]).size
+                    disk['bytes'] += asizeof(value[1])
-                    disk['objects'] += hp.iso(value[1]).count
+                    disk['objects'] += 1
                 disk['entries'] += 1
                 if value[0] < disk['oldest']:
                     disk['oldest'] = value[0]
@@ -511,7 +510,7 @@ def ccache():
         total['keys'] = key_table(total['keys'])
 
     return dict(form=form, total=total,
-                ram=ram, disk=disk, object_stats=hp != False)
+                ram=ram, disk=disk, object_stats=asizeof != False)
 
 
 def table_template(table):
@@ -657,37 +656,36 @@ def d3_graph_model():
     Create a list of table dicts, called "nodes"
     """
     
-    data = {}
     nodes = []
     links = []
 
-    subgraphs = dict()
+    for database in databases:
+        db = eval_in_global_env(database)
+        for tablename in db.tables:
+            fields = []
+            for field in db[tablename]:
+                f_type = field.type
+                if not isinstance(f_type,str):
+                    disp = ' '
+                elif f_type == 'string':
+                    disp =  field.length
+                elif f_type == 'id':
+                    disp =  "PK"
+                elif f_type.startswith('reference') or \
+                    f_type.startswith('list:reference'):
+                    disp = "FK"
+                else:
+                    disp = ' '
+                fields.append(dict(name= field.name, type=field.type, disp = disp))
 
-    for tablename in db.tables:
+                if isinstance(f_type,str) and (
-        fields = []
+                    f_type.startswith('reference') or
-        for field in db[tablename]:
+                    f_type.startswith('list:reference')):
-            f_type = field.type
+                    referenced_table = f_type.split()[1].split('.')[0]
-            if not isinstance(f_type,str):
-                disp = ' '
-            elif f_type == 'string':
-                disp =  field.length
-            elif f_type == 'id':
-                disp =  "PK"
-            elif f_type.startswith('reference') or \
-                f_type.startswith('list:reference'):
-                disp = "FK"
-            else:
-                disp = ' '
-            fields.append(dict(name= field.name, type=field.type, disp = disp))
 
-            if isinstance(f_type,str) and (
+                    links.append(dict(source=tablename, target = referenced_table))
-                f_type.startswith('reference') or
-                f_type.startswith('list:reference')):
-                referenced_table = f_type.split()[1].split('.')[0]
 
-                links.append(dict(source=tablename, target = referenced_table))
+            nodes.append(dict(name=tablename, type="table", fields = fields))
-
-        nodes.append(dict(name=tablename, type="table", fields = fields))
 
     # d3 v4 allows individual modules to be specified.  The complete d3 library is included below.
     response.files.append(URL('admin','static','js/d3.min.js'))

applications/welcome/controllers/default.py

@@ -1,26 +1,35 @@
 # -*- coding: utf-8 -*-
-# this file is released under public domain and you can use without limitations
-
 # -------------------------------------------------------------------------
 # This is a sample controller
-# - index is the default action of any application
+# this file is released under public domain and you can use without limitations
-# - user is required for authentication and authorization
-# - download is for downloading files uploaded in the db (does streaming)
 # -------------------------------------------------------------------------
 
-
+# ---- example index page ----
 def index():
-    """
-    example action using the internationalization operator T and flash
-    rendered by views/default/index.html or views/generic.html
-
-    if you need a simple wiki simply replace the two lines below with:
-    return auth.wiki()
-    """
     response.flash = T("Hello World")
     return dict(message=T('Welcome to web2py!'))
 
+# ---- API (example) -----
+@auth.requires_login()
+def api_get_user_email():
+    if not request.env.request_method == 'GET': raise HTTP(403)
+    return response.json({'status':'success', 'email':auth.user.email})
 
+# ---- Smart Grid (example) -----
+@auth.requires_membership('admin') # can only be accessed by members of admin groupd
+def grid():
+    response.view = 'generic.html' # use a generic view
+    tablename = request.args(0)
+    if not tablename in db.tables: raise HTTP(403)
+    grid = SQLFORM.smartgrid(db[tablename], args=[tablename], deletable=False, editable=False)
+    return dict(grid=grid)
+
+# ---- Embedded wiki (example) ----
+def wiki():
+    auth.wikimenu() # add the wiki to the menu
+    return auth.wiki() 
+
+# ---- Action for login/register/etc (required for auth) -----
 def user():
     """
     exposes:
@@ -39,7 +48,7 @@ def user():
     """
     return dict(form=auth())
 
-
+# ---- action to server uploaded static content (required) ---
 @cache.action()
 def download():
     """
@@ -47,15 +56,3 @@ def download():
     http://..../[app]/default/download/[filename]
     """
     return response.download(request, db)
-
-
-def call():
-    """
-    exposes services. for example:
-    http://..../[app]/default/call/jsonrpc
-    decorate with @services.jsonrpc the functions to expose
-    supports xml, json, xmlrpc, jsonrpc, amfrpc, rss, csv
-    """
-    return service()
-
-

applications/welcome/languages/es.py

@@ -8,8 +8,8 @@
 '%s %%{row} deleted': '%s %%{fila} %%{eliminada}',
 '%s %%{row} updated': '%s %%{fila} %%{actualizada}',
 '%s selected': '%s %%{seleccionado}',
-'%Y-%m-%d': '%d/%m/%A',
+'%Y-%m-%d': '%d/%m/%Y',
-'%Y-%m-%d %H:%M:%S': '%d/%m/%A %H:%M:%S',
+'%Y-%m-%d %H:%M:%S': '%d/%m/%Y %H:%M:%S',
 '(**%.0d MB**)': '(**%.0d MB**)',
 '(something like "it-it")': '(algo como "it-it")',
 '**%(items)s** %%{item(items)}, **%(bytes)s** %%{byte(bytes)}': '**%(items)s** %%{item(items)}, **%(bytes)s** %%{byte(bytes)}',

applications/welcome/models/db.py

@@ -1,12 +1,19 @@
 # -*- coding: utf-8 -*-
 
+# -------------------------------------------------------------------------
+# AppConfig configuration made easy. Look inside private/appconfig.ini
+# Auth is for authenticaiton and access control
+# -------------------------------------------------------------------------
+from gluon.contrib.appconfig import AppConfig
+from gluon.tools import Auth
+
 # -------------------------------------------------------------------------
 # This scaffolding model makes your app work on Google App Engine too
 # File is released under public domain and you can use without limitations
 # -------------------------------------------------------------------------
 
-if request.global_settings.web2py_version < "2.14.1":
+if request.global_settings.web2py_version < "2.15.5":
-    raise HTTP(500, "Requires web2py 2.13.3 or newer")
+    raise HTTP(500, "Requires web2py 2.15.5 or newer")
 
 # -------------------------------------------------------------------------
 # if SSL/HTTPS is properly configured and you want all HTTP requests to
@@ -14,23 +21,18 @@ if request.global_settings.web2py_version < "2.14.1":
 # -------------------------------------------------------------------------
 # request.requires_https()
 
-# -------------------------------------------------------------------------
-# app configuration made easy. Look inside private/appconfig.ini
-# -------------------------------------------------------------------------
-from gluon.contrib.appconfig import AppConfig
-
 # -------------------------------------------------------------------------
 # once in production, remove reload=True to gain full speed
 # -------------------------------------------------------------------------
-myconf = AppConfig(reload=True)
+configuration = AppConfig(reload=True)
 
 if not request.env.web2py_runtime_gae:
     # ---------------------------------------------------------------------
     # if NOT running on Google App Engine use SQLite or other DB
     # ---------------------------------------------------------------------
-    db = DAL(myconf.get('db.uri'),
+    db = DAL(configuration.get('db.uri'),
-             pool_size=myconf.get('db.pool_size'),
+             pool_size=configuration.get('db.pool_size'),
-             migrate_enabled=myconf.get('db.migrate'),
+             migrate_enabled=configuration.get('db.migrate'),
              check_reserved=['all'])
 else:
     # ---------------------------------------------------------------------
@@ -52,12 +54,15 @@ else:
 # by default give a view/generic.extension to all actions from localhost
 # none otherwise. a pattern can be 'controller/function.extension'
 # -------------------------------------------------------------------------
-response.generic_patterns = ['*'] if request.is_local else []
+response.generic_patterns = [] 
+if request.is_local and not configuration.get('app.production'):
+    response.generic_patterns.append('*')
+
 # -------------------------------------------------------------------------
 # choose a style for forms
 # -------------------------------------------------------------------------
-response.formstyle = myconf.get('forms.formstyle')  # or 'bootstrap3_stacked' or 'bootstrap2' or other
+response.formstyle = 'bootstrap4_inline'
-response.form_label_separator = myconf.get('forms.separator') or ''
+response.form_label_separator = ''
 
 # -------------------------------------------------------------------------
 # (optional) optimize handling of static files
@@ -80,27 +85,24 @@ response.form_label_separator = myconf.get('forms.separator') or ''
 # (more options discussed in gluon/tools.py)
 # -------------------------------------------------------------------------
 
-from gluon.tools import Auth, Service, PluginManager
-
 # host names must be a list of allowed host names (glob syntax allowed)
-auth = Auth(db, host_names=myconf.get('host.names'))
+auth = Auth(db, host_names=configuration.get('host.names'))
-service = Service()
-plugins = PluginManager()
 
 # -------------------------------------------------------------------------
-# create all tables needed by auth if not custom tables
+# create all tables needed by auth, maybe add a list of extra fields
 # -------------------------------------------------------------------------
+auth.settings.extra_fields['auth_user'] = []
 auth.define_tables(username=False, signature=False)
 
 # -------------------------------------------------------------------------
 # configure email
 # -------------------------------------------------------------------------
 mail = auth.settings.mailer
-mail.settings.server = 'logging' if request.is_local else myconf.get('smtp.server')
+mail.settings.server = 'logging' if request.is_local else configuration.get('smtp.server')
-mail.settings.sender = myconf.get('smtp.sender')
+mail.settings.sender = configuration.get('smtp.sender')
-mail.settings.login = myconf.get('smtp.login')
+mail.settings.login = configuration.get('smtp.login')
-mail.settings.tls = myconf.get('smtp.tls') or False
+mail.settings.tls = configuration.get('smtp.tls') or False
-mail.settings.ssl = myconf.get('smtp.ssl') or False
+mail.settings.ssl = configuration.get('smtp.ssl') or False
 
 # -------------------------------------------------------------------------
 # configure auth policy
@@ -109,6 +111,27 @@ auth.settings.registration_requires_verification = False
 auth.settings.registration_requires_approval = False
 auth.settings.reset_password_requires_verification = True
 
+# -------------------------------------------------------------------------  
+# read more at http://dev.w3.org/html5/markup/meta.name.html               
+# -------------------------------------------------------------------------
+response.meta.author = configuration.get('app.author')
+response.meta.description = configuration.get('app.description')
+response.meta.keywords = configuration.get('app.keywords')
+response.meta.generator = configuration.get('app.generator')
+response.show_toolbar = configuration.get('app.toolbar')
+
+# -------------------------------------------------------------------------
+# your http://google.com/analytics id                                      
+# -------------------------------------------------------------------------
+response.google_analytics_id = configuration.get('google.analytics_id')
+
+# -------------------------------------------------------------------------
+# maybe use the scheduler
+# -------------------------------------------------------------------------
+if configuration.get('scheduler.enabled'):
+    from gluon.scheduler import Scheduler
+    scheduler = Scheduler(db, heartbeat=configuration.get('heartbeat'))
+
 # -------------------------------------------------------------------------
 # Define your tables below (or better in another model file) for example
 #

applications/welcome/models/menu.py

@@ -1,29 +1,6 @@
 # -*- coding: utf-8 -*-
 # this file is released under public domain and you can use without limitations
 
-# ----------------------------------------------------------------------------------------------------------------------
-# Customize your APP title, subtitle and menus here
-# ----------------------------------------------------------------------------------------------------------------------
-
-response.logo = A(B('web', SPAN(2), 'py'), XML('™ '),
-                  _class="navbar-brand", _href="http://www.web2py.com/",
-                  _id="web2py-logo")
-response.title = request.application.replace('_', ' ').title()
-response.subtitle = ''
-
-# ----------------------------------------------------------------------------------------------------------------------
-# read more at http://dev.w3.org/html5/markup/meta.name.html
-# ----------------------------------------------------------------------------------------------------------------------
-response.meta.author = myconf.get('app.author')
-response.meta.description = myconf.get('app.description')
-response.meta.keywords = myconf.get('app.keywords')
-response.meta.generator = myconf.get('app.generator')
-
-# ----------------------------------------------------------------------------------------------------------------------
-# your http://google.com/analytics id
-# ----------------------------------------------------------------------------------------------------------------------
-response.google_analytics_id = None
-
 # ----------------------------------------------------------------------------------------------------------------------
 # this is the main application menu add/remove items as required
 # ----------------------------------------------------------------------------------------------------------------------
@@ -32,53 +9,42 @@ response.menu = [
     (T('Home'), False, URL('default', 'index'), [])
 ]
 
-DEVELOPMENT_MENU = True
-
-
 # ----------------------------------------------------------------------------------------------------------------------
-# provide shortcuts for development. remove in production
+# provide shortcuts for development. you can remove everything below in production
 # ----------------------------------------------------------------------------------------------------------------------
 
-def _():
+if not configuration.get('app.production'):
-    # ------------------------------------------------------------------------------------------------------------------
+    _app = request.application
-    # shortcuts
-    # ------------------------------------------------------------------------------------------------------------------
-    app = request.application
-    ctr = request.controller
-    # ------------------------------------------------------------------------------------------------------------------
-    # useful links to internal and external resources
-    # ------------------------------------------------------------------------------------------------------------------
     response.menu += [
         (T('My Sites'), False, URL('admin', 'default', 'site')),
         (T('This App'), False, '#', [
-            (T('Design'), False, URL('admin', 'default', 'design/%s' % app)),
+            (T('Design'), False, URL('admin', 'default', 'design/%s' % _app)),
-            LI(_class="divider"),
             (T('Controller'), False,
              URL(
-                 'admin', 'default', 'edit/%s/controllers/%s.py' % (app, ctr))),
+                 'admin', 'default', 'edit/%s/controllers/%s.py' % (_app, request.controller))),
             (T('View'), False,
              URL(
-                 'admin', 'default', 'edit/%s/views/%s' % (app, response.view))),
+                 'admin', 'default', 'edit/%s/views/%s' % (_app, response.view))),
             (T('DB Model'), False,
              URL(
-                 'admin', 'default', 'edit/%s/models/db.py' % app)),
+                 'admin', 'default', 'edit/%s/models/db.py' % _app)),
             (T('Menu Model'), False,
              URL(
-                 'admin', 'default', 'edit/%s/models/menu.py' % app)),
+                 'admin', 'default', 'edit/%s/models/menu.py' % _app)),
             (T('Config.ini'), False,
              URL(
-                 'admin', 'default', 'edit/%s/private/appconfig.ini' % app)),
+                 'admin', 'default', 'edit/%s/private/appconfig.ini' % _app)),
             (T('Layout'), False,
              URL(
-                 'admin', 'default', 'edit/%s/views/layout.html' % app)),
+                 'admin', 'default', 'edit/%s/views/layout.html' % _app)),
             (T('Stylesheet'), False,
              URL(
-                 'admin', 'default', 'edit/%s/static/css/web2py-bootstrap3.css' % app)),
+                 'admin', 'default', 'edit/%s/static/css/web2py-bootstrap3.css' % _app)),
-            (T('Database'), False, URL(app, 'appadmin', 'index')),
+            (T('Database'), False, URL(_app, 'appadmin', 'index')),
             (T('Errors'), False, URL(
-                'admin', 'default', 'errors/' + app)),
+                'admin', 'default', 'errors/' + _app)),
             (T('About'), False, URL(
-                'admin', 'default', 'about/' + app)),
+                'admin', 'default', 'about/' + _app)),
         ]),
         ('web2py.com', False, '#', [
             (T('Download'), False,
@@ -98,7 +64,6 @@ def _():
         ]),
         (T('Documentation'), False, '#', [
             (T('Online book'), False, 'http://www.web2py.com/book'),
-            LI(_class="divider"),
             (T('Preface'), False,
              'http://www.web2py.com/book/default/chapter/00'),
             (T('Introduction'), False,
@@ -143,9 +108,3 @@ def _():
         ]),
     ]
 
-
-if DEVELOPMENT_MENU:
-    _()
-
-if "auth" in locals():
-    auth.wikimenu()

applications/welcome/private/appconfig.ini

@@ -5,6 +5,8 @@ author      = Your Name <you@example.com>
 description = a cool new app
 keywords    = web2py, python, framework
 generator   = Web2py Web Framework
+production  = false
+toolbar     = false
 
 ; Host configuration
 [host]
@@ -14,7 +16,6 @@ names = localhost:*, 127.0.0.1:*, *:*, *
 [db]
 uri       = sqlite://storage.sqlite
 migrate   = true
-; ignored for sqlite
 pool_size = 10  
 
 ; smtp address and credentials
@@ -25,7 +26,9 @@ login  = username:password
 tls    = true
 ssl    = true
 
-; form styling
+[scheduler]
-[forms]
+enabled   = false
-formstyle = bootstrap3_inline
+heartbeat = 1
-separator = 
+
+[google]
+analytics_id =

applications/welcome/static/css/web2py-bootstrap4.css

@@ -1,3 +1,11 @@
+ul.navbar-nav.navbar-right {
+    position: absolute;
+    right: 100px;
+}
+label, th {
+    font-weigth: bold;
+    white-space: nowrap;
+}
 div.w2p_flash {
     background-image: none;
     border-radius: 4px;
@@ -108,7 +116,6 @@ select.autocomplete {
     background: url(../images/background.jpg) no-repeat center center;
 }
 body {
-    padding-top: 60px;
     margin-bottom: 60px;
 }
 header {
@@ -126,8 +133,7 @@ html {
     bottom: 0;
     width: 100%;
     height: 60px;
-    background: #333;
+    background: #f7f7f7;
-    color: #aaa;
 }
 header h1 {
     color: #FFF!important;
@@ -315,3 +321,20 @@ td.w2p_fc,
 input[type=checkbox], input[type=radio] {
     margin: 4px 4px 0 0;
 }
+
+/* for backward compatbility with pre-font-awesome */
+.icon.plus,.icon.arrowleft,.icon.download,.icon.trash,.icon.pen,.icon.arrowright,.icon.magnifier {
+    display: inline-block;
+    font: normal normal normal 14px/1 FontAwesome;
+    font-size: inherit;
+    text-rendering: auto;
+    -webkit-font-smoothing: antialiased;
+    -moz-osx-font-smoothing: grayscale;
+}
+.icon.plus:before { content: "\f067";}
+.icon.arrowleft:before { content: "\f060";}
+.icon.download:before { content: "\f019";}
+.icon.trash:before { content: "\f1f8";}
+.icon.pen:before { content: "\f040";}
+.icon.arrowright:before { content: "\f061";}
+.icon.magnifier:before { content: "\f002";}

applications/welcome/static/js/respond-1.4.2.min.js

@@ -1,6 +0,0 @@
-/*! Respond.js v1.4.2: min/max-width media query polyfill
- * Copyright 2014 Scott Jehl
- * Licensed under MIT
- * http://j.mp/respondjs */
-
-!function(a){"use strict";a.matchMedia=a.matchMedia||function(a){var b,c=a.documentElement,d=c.firstElementChild||c.firstChild,e=a.createElement("body"),f=a.createElement("div");return f.id="mq-test-1",f.style.cssText="position:absolute;top:-100em",e.style.background="none",e.appendChild(f),function(a){return f.innerHTML='&shy;<style media="'+a+'"> #mq-test-1 { width: 42px; }</style>',c.insertBefore(e,d),b=42===f.offsetWidth,c.removeChild(e),{matches:b,media:a}}}(a.document)}(this),function(a){"use strict";function b(){v(!0)}var c={};a.respond=c,c.update=function(){};var d=[],e=function(){var b=!1;try{b=new a.XMLHttpRequest}catch(c){b=new a.ActiveXObject("Microsoft.XMLHTTP")}return function(){return b}}(),f=function(a,b){var c=e();c&&(c.open("GET",a,!0),c.onreadystatechange=function(){4!==c.readyState||200!==c.status&&304!==c.status||b(c.responseText)},4!==c.readyState&&c.send(null))},g=function(a){return a.replace(c.regex.minmaxwh,"").match(c.regex.other)};if(c.ajax=f,c.queue=d,c.unsupportedmq=g,c.regex={media:/@media[^\{]+\{([^\{\}]*\{[^\}\{]*\})+/gi,keyframes:/@(?:\-(?:o|moz|webkit)\-)?keyframes[^\{]+\{(?:[^\{\}]*\{[^\}\{]*\})+[^\}]*\}/gi,comments:/\/\*[^*]*\*+([^/][^*]*\*+)*\//gi,urls:/(url\()['"]?([^\/\)'"][^:\)'"]+)['"]?(\))/g,findStyles:/@media *([^\{]+)\{([\S\s]+?)$/,only:/(only\s+)?([a-zA-Z]+)\s?/,minw:/\(\s*min\-width\s*:\s*(\s*[0-9\.]+)(px|em)\s*\)/,maxw:/\(\s*max\-width\s*:\s*(\s*[0-9\.]+)(px|em)\s*\)/,minmaxwh:/\(\s*m(in|ax)\-(height|width)\s*:\s*(\s*[0-9\.]+)(px|em)\s*\)/gi,other:/\([^\)]*\)/g},c.mediaQueriesSupported=a.matchMedia&&null!==a.matchMedia("only all")&&a.matchMedia("only all").matches,!c.mediaQueriesSupported){var h,i,j,k=a.document,l=k.documentElement,m=[],n=[],o=[],p={},q=30,r=k.getElementsByTagName("head")[0]||l,s=k.getElementsByTagName("base")[0],t=r.getElementsByTagName("link"),u=function(){var a,b=k.createElement("div"),c=k.body,d=l.style.fontSize,e=c&&c.style.fontSize,f=!1;return b.style.cssText="position:absolute;font-size:1em;width:1em",c||(c=f=k.createElement("body"),c.style.background="none"),l.style.fontSize="100%",c.style.fontSize="100%",c.appendChild(b),f&&l.insertBefore(c,l.firstChild),a=b.offsetWidth,f?l.removeChild(c):c.removeChild(b),l.style.fontSize=d,e&&(c.style.fontSize=e),a=j=parseFloat(a)},v=function(b){var c="clientWidth",d=l[c],e="CSS1Compat"===k.compatMode&&d||k.body[c]||d,f={},g=t[t.length-1],p=(new Date).getTime();if(b&&h&&q>p-h)return a.clearTimeout(i),i=a.setTimeout(v,q),void 0;h=p;for(var s in m)if(m.hasOwnProperty(s)){var w=m[s],x=w.minw,y=w.maxw,z=null===x,A=null===y,B="em";x&&(x=parseFloat(x)*(x.indexOf(B)>-1?j||u():1)),y&&(y=parseFloat(y)*(y.indexOf(B)>-1?j||u():1)),w.hasquery&&(z&&A||!(z||e>=x)||!(A||y>=e))||(f[w.media]||(f[w.media]=[]),f[w.media].push(n[w.rules]))}for(var C in o)o.hasOwnProperty(C)&&o[C]&&o[C].parentNode===r&&r.removeChild(o[C]);o.length=0;for(var D in f)if(f.hasOwnProperty(D)){var E=k.createElement("style"),F=f[D].join("\n");E.type="text/css",E.media=D,r.insertBefore(E,g.nextSibling),E.styleSheet?E.styleSheet.cssText=F:E.appendChild(k.createTextNode(F)),o.push(E)}},w=function(a,b,d){var e=a.replace(c.regex.comments,"").replace(c.regex.keyframes,"").match(c.regex.media),f=e&&e.length||0;b=b.substring(0,b.lastIndexOf("/"));var h=function(a){return a.replace(c.regex.urls,"$1"+b+"$2$3")},i=!f&&d;b.length&&(b+="/"),i&&(f=1);for(var j=0;f>j;j++){var k,l,o,p;i?(k=d,n.push(h(a))):(k=e[j].match(c.regex.findStyles)&&RegExp.$1,n.push(RegExp.$2&&h(RegExp.$2))),o=k.split(","),p=o.length;for(var q=0;p>q;q++)l=o[q],g(l)||m.push({media:l.split("(")[0].match(c.regex.only)&&RegExp.$2||"all",rules:n.length-1,hasquery:l.indexOf("(")>-1,minw:l.match(c.regex.minw)&&parseFloat(RegExp.$1)+(RegExp.$2||""),maxw:l.match(c.regex.maxw)&&parseFloat(RegExp.$1)+(RegExp.$2||"")})}v()},x=function(){if(d.length){var b=d.shift();f(b.href,function(c){w(c,b.href,b.media),p[b.href]=!0,a.setTimeout(function(){x()},0)})}},y=function(){for(var b=0;b<t.length;b++){var c=t[b],e=c.href,f=c.media,g=c.rel&&"stylesheet"===c.rel.toLowerCase();e&&g&&!p[e]&&(c.styleSheet&&c.styleSheet.rawCssText?(w(c.styleSheet.rawCssText,e,f),p[e]=!0):(!/^([a-zA-Z:]*\/\/)/.test(e)&&!s||e.replace(RegExp.$1,"").split("/")[0]===a.location.host)&&("//"===e.substring(0,2)&&(e=a.location.protocol+e),d.push({href:e,media:f})))}x()};y(),c.update=y,c.getEmValue=u,a.addEventListener?a.addEventListener("resize",b,!1):a.attachEvent&&a.attachEvent("onresize",b)}}(this);

applications/welcome/static/js/share.js

@@ -1,44 +0,0 @@
-/**
-
-   Created and copyrighted by Massimo Di Pierro <massimo.dipierro@gmail.com>
-   (MIT license)  
-
-   Example:
-
-   <script src="share.js"></script>
-
-**/
-
-jQuery(function(){
-	var script_source = jQuery('script[src*="share.js"]').attr('src');
-        var params = function(name,default_value) {
-            var match = RegExp('[?&]' + name + '=([^&]*)').exec(script_source);
-            return match && decodeURIComponent(match[1].replace(/\+/g, ' '))||default_value;
-        }
-	var path = params('static','social');
-	var url = encodeURIComponent(window.location.href);
-	var host =  window.location.hostname;
-	var title = escape(jQuery('title').text());
-	var twit = 'http://twitter.com/home?status='+title+'%20'+url;
-	var facebook = 'http://www.facebook.com/sharer.php?u='+url;
-	var gplus = 'https://plus.google.com/share?url='+url;
-	var tbar = '<div id="socialdrawer"><span>Share<br/></span><div id="sicons"><a href="'+twit+'" id="twit" title="Share on twitter"><img src="'+path+'/twitter.png"  alt="Share on Twitter" width="32" height="32" /></a><a href="'+facebook+'" id="facebook" title="Share on Facebook"><img src="'+path+'/facebook.png"  alt="Share on facebook" width="32" height="32" /></a><a href="'+gplus+'" id="gplus" title="Share on Google Plus"><img src="'+path+'/gplus-32.png"  alt="Share on Google Plus" width="32" height="32" /></a></div></div>';	
-	// Add the share tool bar.
-	jQuery('body').append(tbar); 
-	var st = jQuery('#socialdrawer');
-	st.css({'opacity':'.7','z-index':'3000','background':'#FFF','border':'solid 1px #666','border-width':' 1px 0 0 1px','height':'20px','width':'40px','position':'fixed','bottom':'0','right':'0','padding':'2px 5px','overflow':'hidden','-webkit-border-top-left-radius':' 12px','-moz-border-radius-topleft':' 12px','border-top-left-radius':' 12px','-moz-box-shadow':' -3px -3px 3px rgba(0,0,0,0.5)','-webkit-box-shadow':' -3px -3px 3px rgba(0,0,0,0.5)','box-shadow':' -3px -3px 3px rgba(0,0,0,0.5)'});
-	jQuery('#socialdrawer a').css({'float':'left','width':'32px','margin':'3px 2px 2px 2px','padding':'0','cursor':'pointer'});
-	jQuery('#socialdrawer span').css({'float':'left','margin':'2px 3px','text-shadow':' 1px 1px 1px #FFF','color':'#444','font-size':'12px','line-height':'1em'});
-        jQuery('#socialdrawer img').hide();
-	// hover
-	st.click(function(){
-		jQuery(this).animate({height:'40px', width:'160px', opacity: 0.95}, 300);
-		jQuery('#socialdrawer img').show();
-	    });
-	//leave
-	st.mouseleave(function(){ 
-	    st.animate({height:'20px', width: '40px', opacity: .7}, 300); 
-	    jQuery('#socialdrawer img').hide();
-	    return false;
-	    }  );
-    });

applications/welcome/static/js/web2py.js

@@ -13,6 +13,8 @@
     }
 
     var FORMDATA_IS_SUPPORTED = typeof(FormData) !== 'undefined';
+    var animateIn = 'fadeIn';
+    // var animateIn = 'slideDown';
 
     String.prototype.reverse = function () {
         return this.split('').reverse().join('');
@@ -180,7 +182,7 @@
         },
         /* manage errors in forms */
         manage_errors: function (target) {
-            $('div.error', target).hide().slideDown('slow');
+            $('div.error', target).hide()[animateIn]('slow');
         },
         after_ajax: function (xhr) {
             /* called whenever an ajax request completes */
@@ -265,13 +267,17 @@
                 }
             });
             /* help preventing double form submission for normal form (not LOADed) */
-            $(doc).on('submit', 'form', function () {
+            $(doc).on('submit', 'form', function (e) {
-                var submit_button = $(this).find(web2py.formInputClickSelector);
+                var submit_buttons = $(this).find(web2py.formInputClickSelector);
-                web2py.disableElement(submit_button);
+                submit_buttons.each(function() {
+                    web2py.disableElement($(this));
+                })
                 /* safeguard in case the form doesn't trigger a refresh,
                 see https://github.com/web2py/web2py/issues/1100 */
                 setTimeout(function () {
-                    web2py.enableElement(submit_button);
+                    submit_buttons.each(function() {
+                        web2py.enableElement($(this));
+                    });
                 }, 5000);
             });
             doc.ajaxSuccess(function (e, xhr) {
@@ -611,8 +617,8 @@
         flash: function (message, status) {
             var flash = $('.w2p_flash');
             web2py.hide_flash();
-            flash.html(message).addClass(status);
+            flash.text(message).addClass(status);
-            if (flash.html()) flash.append('<span id="closeflash"> &times; </span>').slideDown();
+            if (flash.html()) flash.append('<span id="closeflash"> &times; </span>')[animateIn]();
         },
         hide_flash: function () {
             $('.w2p_flash').fadeOut(0).html('');
@@ -626,7 +632,7 @@
                 for (var k = 0; k < triggers[id].length; k++) {
                     var dep = $('#' + triggers[id][k], target);
                     var tr = $('#' + triggers[id][k] + '__row', target);
-                    if (t.is(dep.attr('data-show-if'))) tr.slideDown();
+                    if (t.is(dep.attr('data-show-if'))) tr[animateIn]();
                     else tr.hide();
                 }
             };

applications/welcome/views/appadmin.html

@@ -148,7 +148,7 @@
           {{=T.M("(**%.0d MB**)", total['bytes'] / 1048576)}}
         {{pass}}
       {{else:}}
-        {{=T.M("**not available** (requires the Python [[guppy http://pypi.python.org/pypi/guppy/ popup]] library)")}}
+        {{=T.M("**not available** (requires the Python [[Pympler https://pypi.python.org/pypi/Pympler popup]] library)")}}
       {{pass}}
     </p>
     <p>
@@ -176,7 +176,7 @@
           {{=T.M("(**%.0d MB**)", ram['bytes'] / 10485576)}}
         {{pass}}
       {{else:}}
-        {{=T.M("``**not available**``:red (requires the Python [[guppy http://pypi.python.org/pypi/guppy/ popup]] library)")}}
+        {{=T.M("``**not available**``:red (requires the Python [[Pympler https://pypi.python.org/pypi/Pympler popup]] library)")}}
       {{pass}}
     </p>
     <p>
@@ -205,7 +205,7 @@
           {{=T.M("(**%.0d MB**)", disk['bytes'] / 1048576)}}
         {{pass}}
       {{else:}}
-        {{=T.M("``**not available**``:red (requires the Python [[guppy http://pypi.python.org/pypi/guppy/ popup]] library)")}}
+        {{=T.M("``**not available**``:red (requires the Python [[Pympler https://pypi.python.org/pypi/Pympler popup]] library)")}}
       {{pass}}
       </p>
       <p>

applications/welcome/views/default/index.html

@@ -1,15 +1,9 @@
-{{left_sidebar_enabled,right_sidebar_enabled=False,('message' in globals())}}
 {{extend 'layout.html'}}
 
 {{block header}}
-    <header class="container-fluid background">
+<center style="background-color: #333; color:white; padding:30px">
-      <div class="jumbotron text-center">
+  <h1>/{{=request.application}}/{{=request.controller}}/{{=request.function}}
-        {{if response.title:}}
+</center>
-        <h1>{{=response.title}}
-          <small>{{=response.subtitle or ''}}</small></h1>
-        {{pass}}
-      </div>
-    </header>
 {{end}}
 
 {{if 'message' in globals():}}
@@ -27,26 +21,23 @@
     _href=URL('admin','default','peek',args=(request.application,'views',request.controller,'index.html')))))}}</li>
   <li>{{=T('You can modify this application and adapt it to your needs')}}</li>
 </ol>
+<center style="padding:50px">
+  <a class="btn btn-primary" href="{{=URL('admin','default','index')}}">
+    <i class="fa fa-cog"></i>
+    {{=T("admin")}}
+  </a>
+  <a class="btn btn-secondary" href="{{=URL('examples','default','index')}}">{{=T("Online examples")}}</a>
+  <a class="btn btn-secondary" href="http://web2py.com">web2py.com</a>
+  <a class="btn btn-secondary" href="http://web2py.com/book">{{=T('Documentation')}}</a>
+  <a class="btn btn-secondary" href="{{=URL('default','api_get_user_email')}}">{{=T('API Example')}}</a>
+  <a class="btn btn-secondary" href="{{=URL('default','grid/auth_user')}}">{{=T('Grid Example')}}</a>
+  <a class="btn btn-secondary" href="{{=URL('default','wiki')}}">{{=T('Wiki Example')}}</a>
+</center>
 {{elif 'content' in globals():}}
 {{=content}}
 {{else:}}
 {{=BEAUTIFY(response._vars)}}
 {{pass}}
 
-{{block right_sidebar}}
+
-<div class="panel panel-info">
+
-  <div class="panel-heading"><h3 class="panel-title"><a class="btn-block"
-      href="{{=URL('admin','default','index')}}">
-      <i class="glyphicon glyphicon-cog"></i>
-      {{=T("admin")}}
-    </a></h3></div>
-  <div class="panel-body">
-    {{=T("Don't know what to do?")}}
-  </div>
-  <ul class="list-group">
-    <li class="list-group-item">{{=A(T("Online examples"), _href=URL('examples','default','index'))}}</li>
-    <li class="list-group-item"><a href="http://web2py.com">web2py.com</a></li>
-    <li class="list-group-item"><a href="http://web2py.com/book">{{=T('Documentation')}}</a></li>
-  </ul>
-</div>
-{{end}}

applications/welcome/views/default/user.html

@@ -10,10 +10,10 @@
         {{
         if request.args(0)=='login':
             if not 'register' in auth.settings.actions_disabled:
-                form.add_button(T('Sign Up'),URL(args='register', vars={'_next': request.vars._next} if request.vars._next else None),_class='btn btn-default')
+                form.add_button(T('Sign Up'),URL(args='register', vars={'_next': request.vars._next} if request.vars._next else None),_class='btn btn-default btn-secondary')
             pass
             if not 'request_reset_password' in auth.settings.actions_disabled:
-                form.add_button(T('Lost Password'),URL(args='request_reset_password'),_class='btn btn-default')
+                form.add_button(T('Lost Password'),URL(args='request_reset_password'),_class='btn btn-default btn-secondary')
             pass
         pass
         =form

applications/welcome/views/generic.html

@@ -11,6 +11,3 @@ It is used as default when a view is not provided for your controllers
 {{elif len(response._vars)>1:}}
 {{=BEAUTIFY(response._vars)}}
 {{pass}}
-{{if request.is_local:}}
-{{=response.toolbar()}}
-{{pass}}

applications/welcome/views/layout.html

@@ -20,85 +20,83 @@
          http://google.com/webmasters -->
     <meta name="google-site-verification" content="">
     <!-- include stylesheets -->
+    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css"/>
     <link rel="stylesheet" href="{{=URL('static','css/bootstrap.min.css')}}"/>
-    <link rel="stylesheet" href="{{=URL('static','css/web2py-bootstrap3.css')}}"/>
+    <link rel="stylesheet" href="{{=URL('static','css/web2py-bootstrap4.css')}}"/>
     <link rel="shortcut icon" href="{{=URL('static','images/favicon.ico')}}" type="image/x-icon">
     <link rel="apple-touch-icon" href="{{=URL('static','images/favicon.png')}}">
     <!-- All JavaScript at the bottom, except for Modernizr which enables
          HTML5 elements & feature detects -->
     <script src="{{=URL('static','js/modernizr-2.8.3.min.js')}}"></script>
-    <!--[if lt IE 9]>
-        <script src="{{=URL('static','js/respond-1.4.2.min.js')}}"></script>
-        <![endif]-->
     <!-- Favicons -->
     {{include 'web2py_ajax.html'}} <!-- this includes jquery.js, calendar.js/.css and web2py.js -->
     {{block head}}{{end}}
-    {{
-    # using sidebars need to know what sidebar you want to use
-    mc0 = 'col-md-12'
-    mc1 = 'col-md-9'
-    mc2 = 'col-md-6'
-    left_sidebar_enabled = globals().get('left_sidebar_enabled', False)
-    right_sidebar_enabled = globals().get('right_sidebar_enabled', False)
-    middle_column = {0: mc0, 1: mc1, 2: mc2}[
-    (left_sidebar_enabled and 1 or 0)+(right_sidebar_enabled and 1 or 0)]
-    }}
   </head>
   <body>
-    <!--[if lt IE 8]><p class="browserupgrade">You are using an <strong>outdated</strong> browser. Please <a href="http://browsehappy.com/">upgrade your browser</a> to improve your experience.</p><![endif]-->
     <div class="w2p_flash alert alert-dismissable">{{=response.flash or ''}}</div>
     <!-- Navbar ======================================= -->
-    <nav class="navbar navbar-default navbar-fixed-top" role="navigation">
+    <nav class="navbar navbar-toggleable-md navbar-light bg-faded">
-      <div class="container-fluid">
+      <button class="navbar-toggler navbar-toggler-right" type="button" data-toggle="collapse" data-target="#navbarNavDropdown" aria-controls="navbarNavDropdown" aria-expanded="false" aria-label="Toggle navigation">
-        <div class="navbar-header">
+        <span class="navbar-toggler-icon"></span>
-          <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse">
+      </button>
-            <span class="sr-only">Toggle navigation</span>
+      <a class="navbar-brand" href="http://web2py.com">web2py</a>
-            <span class="icon-bar"></span>
+      <div id="navbarNavDropdown" class="collapse navbar-collapse">
-            <span class="icon-bar"></span>
+        <ul class="navbar-nav">
-            <span class="icon-bar"></span>
+          {{for _item in response.menu or []:}}
-          </button>
+          {{if len(_item)<4 or not _item[3]:}}
-          {{=response.logo or ''}}
+          <li class="nav-item {{if _item[1]:}}active{{pass}}">
-        </div>
+            <a class="nav-link" href="{{=_item[2]}}">{{=_item[0]}}</a>
-        <div class="collapse navbar-collapse">
+          </li>
-          <ul class="nav navbar-nav navbar-right">
+          {{else:}}
-            {{='auth' in globals() and auth.navbar('Welcome',mode='dropdown') or ''}}
+          <li class="nav-item dropdown">
-          </ul>
+            <a class="nav-link dropdown-toggle" href="{{=_item[2]}}" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">{{=_item[0]}}</a>
-          {{if response.menu:}}
+            <div class="dropdown-menu">
-          {{=MENU(response.menu, _class='nav navbar-nav',li_class='dropdown',ul_class='dropdown-menu')}}
+              {{for _subitem in _item[3]:}}
+              <a class="dropdown-item" href="{{=_subitem[2]}}">{{=_subitem[0]}}</a>
+              {{pass}}
+            </div>
+          </li>
           {{pass}}
-        </div>
+          {{pass}}          
+        </ul>
+        <form class="form-inline my-2 my-lg-0">
+          <input class="form-control mr-sm-2" type="text" placeholder="Search">
+        </form>      
+        {{if 'auth' in globals():}}
+        <ul class="navbar-nav navbar-right">
+          <li class="nav-item dropdown">
+            <a class="nav-link dropdown-toggle" href="#" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">
+              {{if auth.user:}}{{=auth.user.first_name}}{{else:}}LOGIN{{pass}}
+            </a>
+            <div class="dropdown-menu">
+              {{if auth.user:}}
+              <a class="dropdown-item" href="{{=URL('default','user/profile')}}">{{=T('Profile')}}</a>
+              <a class="dropdown-item" href="{{=URL('default','user/change_password')}}">{{=T('Change Password')}}</a>
+              <a class="dropdown-item" href="{{=URL('default','user/logout')}}">{{=T('Logout')}}</a>
+              {{else:}}
+              <a class="dropdown-item" href="{{=URL('default','user/login')}}">{{=T('Login')}}</a>
+              <a class="dropdown-item" href="{{=URL('default','user/register')}}">{{=T('Sign up')}}</a>
+              <a class="dropdown-item" href="{{=URL('default','user/retrieve_password')}}">{{=T('Lost Password')}}</a>
+              {{pass}}
+            </div>
+          </li>
+        </ul>        
+        {{pass}}
       </div>
     </nav>
+
     <!-- Masthead ===================================== -->    
     {{block header}}
     {{end}}
     <!-- Main ========================================= -->
     <!-- Begin page content -->
     <div class="container-fluid main-container">
-      {{if left_sidebar_enabled:}}
+      <div class="col-md-12">
-        <div class="col-md-3 left-sidebar">
-            {{block left_sidebar}}
-            <h3>Left Sidebar</h3>
-            <p></p>
-            {{end}}
-        </div>
-      {{pass}}
-
-      <div class="{{=middle_column}}">
         {{block center}}
         {{include}}
         {{end}}
+        {{=response.toolbar() if response.show_toolbar else ''}}
       </div>
-
-      {{if right_sidebar_enabled:}}
-        <div class="col-md-3">
-            {{block right_sidebar}}
-            <h3>Right Sidebar</h3>
-            <p></p>
-            {{end}}
-        </div>
-      {{pass}}
-
     </div>
 
     {{block footer}} <!-- this is default footer -->      
@@ -114,7 +112,7 @@
     {{end}}
     <!-- The javascript =============================== -->
     <script src="{{=URL('static','js/bootstrap.min.js')}}"></script>
-    <script src="{{=URL('static','js/web2py-bootstrap3.js')}}"></script>
+    <script src="{{=URL('static','js/web2py-bootstrap4.js')}}"></script>
     {{block page_js}}{{end page_js}}
     {{if response.google_analytics_id:}}
     <!-- Analytics ==================================== -->
@@ -125,7 +123,5 @@
       });
     </script>
     {{pass}}
-    <!-- Share ============================y============ -->
-    <script src="{{=URL('static','js/share.js',vars=dict(static=URL('static','images')))}}"></script>
   </body>
 </html>

appveyor.yml

@@ -10,6 +10,10 @@ environment:
       COVERAGE_PROCESS_START: gluon/tests/coverage.ini
       PYTHON_ARCH: "64"
 
+    - PYTHON: "C:/Python36"
+      COVERAGE_PROCESS_START: gluon/tests/coverage.ini
+      PYTHON_ARCH: "64"
+
 clone_depth: 50
 
 init:
@@ -20,7 +24,6 @@ install:
   - python -m ensurepip
   - pip install codecov
   - git submodule update --init --recursive
-  - pip install pycrypto
   # Check that we have the expected version and architecture for Python
   - "python --version"
   - "python -c \"import struct; print(struct.calcsize('P') * 8)\""

gluon/_compat.py

@@ -26,7 +26,7 @@ if PY2:
     from email.MIMEText import MIMEText
     from email.Charset import add_charset, QP as charset_QP
     from urllib import FancyURLopener, urlencode, urlopen
-    from urllib import quote as urllib_quote, unquote as urllib_unquote
+    from urllib import quote as urllib_quote, unquote as urllib_unquote, quote_plus as urllib_quote_plus
     from string import maketrans
     from types import ClassType
     import cgi
@@ -35,6 +35,7 @@ if PY2:
     from gluon.contrib import ipaddress
     BytesIO = StringIO
     reduce = reduce
+    reload = reload
     hashlib_md5 = hashlib.md5
     iterkeys = lambda d: d.iterkeys()
     itervalues = lambda d: d.itervalues()
@@ -63,7 +64,7 @@ if PY2:
             return None
         if isinstance(obj, (bytes, bytearray, buffer)):
             return bytes(obj)
-        if isinstance(obj, unicode):
+        if hasattr(obj, 'encode'):
             return obj.encode(charset, errors)
         raise TypeError('Expected bytes')
 
@@ -77,6 +78,7 @@ else:
     import pickle
     from io import StringIO, BytesIO
     import copyreg
+    from importlib import reload
     from functools import reduce
     from html.parser import HTMLParser
     from http import cookies as Cookie
@@ -94,7 +96,7 @@ else:
     from email.header import Header
     from email.charset import Charset, add_charset, QP as charset_QP
     from urllib.request import FancyURLopener, urlopen
-    from urllib.parse import quote as urllib_quote, unquote as urllib_unquote, urlencode
+    from urllib.parse import quote as urllib_quote, unquote as urllib_unquote, urlencode, quote_plus as urllib_quote_plus
     from http import cookiejar as cookielib
     from xmlrpc.client import ProtocolError
     import html # warning, this is the python3 module and not the web2py html module
@@ -122,7 +124,7 @@ else:
             return None
         if isinstance(obj, (bytes, bytearray, memoryview)):
             return bytes(obj)
-        if isinstance(obj, str):
+        if hasattr(obj, 'encode'):
             return obj.encode(charset, errors)
         raise TypeError('Expected bytes')
 
@@ -151,7 +153,7 @@ def with_metaclass(meta, *bases):
 def to_unicode(obj, charset='utf-8', errors='strict'):
     if obj is None:
         return None
-    if not isinstance(obj, bytes):
+    if not hasattr(obj, 'decode'):
         return text_type(obj)
     return obj.decode(charset, errors)
 

gluon/authapi.py

@@ -557,27 +557,43 @@ class AuthAPI(object):
         self.log_event(self.messages['del_membership_log'],
                        dict(user_id=user_id, group_id=group_id))
         ret = self.db(membership.user_id == user_id)(membership.group_id == group_id).delete()
-        if group_id in self.user_groups:
+        if group_id in self.user_groups and user_id == self.user_id:
             del self.user_groups[group_id]
         return ret
 
-    def has_membership(self, group_id=None, user_id=None, role=None):
+    def has_membership(self, group_id=None, user_id=None, role=None, cached=False):
         """
         Checks if user is member of group_id or role
+
+        NOTE: To avoid database query at each page load that use auth.has_membership, someone can use cached=True.
+              If cached is set to True has_membership() check group_id or role only against auth.user_groups variable
+              which is populated properly only at login time. This means that if an user membership change during a
+              given session the user has to log off and log in again in order to auth.user_groups to be properly
+              recreated and reflecting the user membership modification. There is one exception to this log off and
+              log in process which is in case that the user change his own membership, in this case auth.user_groups
+              can be properly update for the actual connected user because web2py has access to the proper session
+              user_groups variable. To make use of this exception someone has to place an "auth.update_groups()"
+              instruction in his app code to force auth.user_groups to be updated. As mention this will only work if it
+              the user itself that change it membership not if another user, let say an administrator, change someone
+              else's membership.
         """
-        group_id = group_id or self.id_group(role)
-        try:
-            group_id = int(group_id)
-        except:
-            group_id = self.id_group(group_id)  # interpret group_id as a role
         if not user_id and self.user:
             user_id = self.user.id
-        membership = self.table_membership()
+        if cached:
-        if group_id and user_id and self.db((membership.user_id == user_id) &
+            id_role = group_id or role
-                                            (membership.group_id == group_id)).select():
+            r = (user_id and id_role in self.user_groups.values()) or (user_id and id_role in self.user_groups)
-            r = True
         else:
-            r = False
+            group_id = group_id or self.id_group(role)
+            try:
+                group_id = int(group_id)
+            except:
+                group_id = self.id_group(group_id)  # interpret group_id as a role
+            membership = self.table_membership()
+            if group_id and user_id and self.db((membership.user_id == user_id) &
+                                                (membership.group_id == group_id)).select():
+                r = True
+            else:
+                r = False
         self.log_event(self.messages['has_membership_log'],
                        dict(user_id=user_id, group_id=group_id, check=r))
         return r
@@ -972,7 +988,8 @@ class AuthAPI(object):
             requires = [requires]
         requires = list(filter(lambda t: isinstance(t, CRYPT), requires))
         if requires:
-            requires[0].min_length = 0
+            requires[0] = CRYPT(**requires[0].__dict__) # Copy the existing CRYPT attributes
+            requires[0].min_length = 0 # But do not enforce minimum length for the old password
 
         old_password = kwargs.get('old_password', '')
         new_password = kwargs.get('new_password', '')

gluon/compileapp.py

@@ -18,7 +18,7 @@ import fnmatch
 import os
 import copy
 import random
-from gluon._compat import builtin, PY2, unicodeT, to_native, to_bytes, iteritems, basestring, reduce, xrange, long
+from gluon._compat import builtin, PY2, unicodeT, to_native, to_bytes, iteritems, basestring, reduce, xrange, long, reload
 from gluon.storage import Storage, List
 from gluon.template import parse_template
 from gluon.restricted import restricted, compile2
@@ -205,7 +205,7 @@ def LOAD(c=None, f='index', args=None, vars=None,
         other_response = Response()
         other_request.env.path_info = '/' + \
             '/'.join([request.application, c, f] +
-                     map(str, other_request.args))
+                     [str(a) for a in other_request.args])
         other_request.env.query_string = \
             vars and URL(vars=vars).split('?')[1] or ''
         other_request.env.http_web2py_component_location = \
@@ -288,7 +288,7 @@ class LoadFactory(object):
             other_response = globals.Response()
             other_request.env.path_info = '/' + \
                 '/'.join([request.application, c, f] +
-                         map(str, other_request.args))
+                         [str(a) for a in other_request.args])
             other_request.env.query_string = \
                 vars and html.URL(vars=vars).split('?')[1] or ''
             other_request.env.http_web2py_component_location = \
@@ -676,8 +676,9 @@ def run_view_in(environment):
     badv = 'invalid view (%s)' % view
     patterns = response.get('generic_patterns')
     layer = None
+    scode = None
     if patterns:
-        regex = re_compile('|'.join(map(fnmatch.translate, patterns)))
+        regex = re_compile('|'.join(fnmatch.translate(p) for p in patterns))
         short_action = '%(controller)s/%(function)s.%(extension)s' % request
         allow_generic = regex.search(short_action)
     else:
@@ -708,22 +709,23 @@ def run_view_in(environment):
                         ccode = getcfs(compiled, compiled, lambda: read_pyc(compiled))
                         layer = compiled
                         break
-        if not os.path.exists(filename) and allow_generic:
-            view = 'generic.' + request.extension
-            filename = pjoin(folder, 'views', view)
-        if not os.path.exists(filename):
-            raise HTTP(404,
-                       rewrite.THREAD_LOCAL.routes.error_message % badv,
-                       web2py_error=badv)
-
         # if the view is not compiled
         if not layer:
-            # Compile the template
+            if not os.path.exists(filename) and allow_generic:
-            ccode = parse_template(view,
+                view = 'generic.' + request.extension
+                filename = pjoin(folder, 'views', view)
+            if not os.path.exists(filename):
+                raise HTTP(404,
+                           rewrite.THREAD_LOCAL.routes.error_message % badv,
+                           web2py_error=badv)
+            # Parse template
+            scode = parse_template(view,
                                    pjoin(folder, 'views'),
                                    context=environment)
-        layer = filename
+            # Compile template
-    restricted(ccode, environment, layer=layer)
+            ccode = compile2(scode, filename)
+            layer = filename        
+    restricted(ccode, environment, layer=layer, scode=scode)
     # parse_template saves everything in response body
     return environment['response'].body.getvalue()
 

gluon/contrib/login_methods/cas_auth.py

@@ -49,7 +49,8 @@ class CasAuth(object):
                            email=lambda v: v.get('email', None),
                            user_id=lambda v: v['user']),
                  casversion=1,
-                 casusername='cas:user'
+                 casusername='cas:user',
+		 change_password_url=None
                  ):
         self.urlbase = urlbase
         self.cas_login_url = "%s/%s" % (self.urlbase, actions[0])
@@ -64,6 +65,9 @@ class CasAuth(object):
                               #vars=current.request.vars,
                               scheme=True)
 
+	# URL to let users change their password in the IDP system
+        self.cas_change_password_url = change_password_url
+
     def login_url(self, next="/"):
         current.session.token = self._CAS_login()
         return next
@@ -74,6 +78,10 @@ class CasAuth(object):
         self._CAS_logout()
         return next
 
+    def change_password_url(self, next="/"):
+        self._CAS_change_password()
+        return next
+
     def get_user(self):
         user = current.session.token
         if user:
@@ -135,3 +143,6 @@ class CasAuth(object):
         redirects to the CAS logout page
         """
         redirect("%s?service=%s" % (self.cas_logout_url, self.cas_my_url))
+
+    def _CAS_change_password(self):
+        redirect(self.cas_change_password_url)

gluon/contrib/login_methods/janrain_account.py

@@ -78,10 +78,13 @@ class RPXAccount(object):
 
     def get_user(self):
         request = self.request
-        if request.vars.token:
+        # Janrain now sends the token via both a POST body and the query
+        # string, so we should keep only one of these.
+        token = request.post_vars.token or request.get_vars.token
+        if token:
             user = Storage()
             data = urllib.urlencode(
-                dict(apiKey=self.api_key, token=request.vars.token))
+                dict(apiKey=self.api_key, token=token))
             auth_info_json = fetch(self.auth_url + '?' + data)
             auth_info = json.loads(auth_info_json)
 

gluon/contrib/login_methods/rpx_account.py

@@ -13,12 +13,11 @@
 
 import os
 import re
-import urllib
 from gluon import *
 from gluon.tools import fetch
 from gluon.storage import Storage
 import json
-
+from gluon._compat import urlencode
 
 class RPXAccount(object):
 
@@ -78,10 +77,13 @@ class RPXAccount(object):
 
     def get_user(self):
         request = self.request
-        if request.vars.token:
+        # Janrain now sends the token via both a POST body and the query
+        # string, so we should keep only one of these.
+        token = request.post_vars.token or request.get_vars.token
+        if token:
             user = Storage()
-            data = urllib.urlencode(
+            data = urlencode(
-                dict(apiKey=self.api_key, token=request.vars.token))
+                dict(apiKey=self.api_key, token=token))
             auth_info_json = fetch(self.auth_url + '?' + data)
             auth_info = json.loads(auth_info_json)
 

gluon/contrib/login_methods/saml2_auth.py

@@ -104,11 +104,12 @@ def obj2dict(obj, processed=None):
                                     types.BuiltinFunctionType,
                                     types.BuiltinMethodType))
 
-def saml2_handler(session, request, config_filename = None):
+def saml2_handler(session, request, config_filename = None, entityid = None):
     config_filename = config_filename or os.path.join(request.folder,'private','sp_conf')
     client = Saml2Client(config_file = config_filename)
-    idps = client.metadata.with_descriptor("idpsso")
+    if not entityid:
-    entityid = idps.keys()[0]
+    	idps = client.metadata.with_descriptor("idpsso")
+    	entityid = idps.keys()[0]
     bindings = [BINDING_HTTP_REDIRECT, BINDING_HTTP_POST]
     binding, destination = client.pick_binding(
         "single_sign_on_service", bindings, "idpsso", entity_id=entityid)
@@ -145,12 +146,21 @@ class Saml2Auth(object):
             username=lambda v:v['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/upn'][0],
             email=lambda v:v['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/upn'][0],
             user_id=lambda v:v['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/upn'][0],
-            )):
+            ), logout_url=None, change_password_url=None, entityid=None):
         self.config_file = config_file
         self.maps = maps
 
+	# URL for redirecting users to when they sign out
+        self.saml_logout_url = logout_url
+
+        # URL to let users change their password in the IDP system
+        self.saml_change_password_url = change_password_url
+	
+	# URL to specify an IDP if using federation metadata or an MDQ
+	self.entityid = entityid
+
     def login_url(self, next="/"):
-        d = saml2_handler(current.session, current.request)
+        d = saml2_handler(current.session, current.request, entityid=self.entityid)
         if 'url' in d:
             redirect(d['url'])
         elif 'error' in d:
@@ -170,6 +180,12 @@ class Saml2Auth(object):
 
     def logout_url(self, next="/"):
         current.session.saml2_info = None
+        current.session.auth = None
+        self._SAML_logout()
+        return next
+
+    def change_password_url(self, next="/"):
+        self._SAML_change_password()
         return next
 
     def get_user(self):        
@@ -180,3 +196,13 @@ class Saml2Auth(object):
                 d[key] = self.maps[key](user)
             return d
         return None
+
+    def _SAML_logout(self):
+        """
+        exposed SAML.logout()
+        redirects to the SAML logout page
+        """
+        redirect(self.saml_logout_url)
+
+    def _SAML_change_password(self):
+        redirect(self.saml_change_password_url)

gluon/contrib/populate.py

@@ -137,6 +137,8 @@ def populate_generator(table, default=True, compute=False, contents={}):
                 continue
             elif field.type == 'upload':
                 continue
+            elif field.compute is not None:
+                continue
             elif default and not field.default in (None, ''):
                 record[fieldname] = field.default
             elif compute and field.compute:

gluon/contrib/user_agent_parser.py

@@ -673,3 +673,23 @@ def simple_detect(agent):
     if os_version:
         os = " ".join((os, os_version))
     return os, browser
+
+
+class mobilize(object):  
+    """
+    Decorator for controller functions so they use different views for mobile devices.
+
+    WARNING: If you update httpagentparser make sure to leave mobilize for
+    backwards compatibility.
+    """
+    def __init__(self, func):
+        self.func = func
+    
+    def __call__(self):
+        from gluon import current
+        user_agent = current.request.user_agent()
+        if user_agent.is_mobile:
+            items = current.response.view.split('.')
+            items.insert(-1, 'mobile')
+            current.response.view = '.'.join(items)
+        return self.func()

gluon/contrib/websocket_messaging.py

@@ -94,32 +94,10 @@ import optparse
 import time
 import sys
 import gluon.utils
-
+from gluon._compat import to_native, to_bytes, urlencode, urlopen
-if (sys.version_info[0] == 2):
-    from urllib import urlencode, urlopen
-    def to_bytes(obj, charset='utf-8', errors='strict'):
-        if obj is None:
-            return None
-        if isinstance(obj, (bytes, bytearray, buffer)):
-            return bytes(obj)
-        if isinstance(obj, unicode):
-            return obj.encode(charset, errors)
-        raise TypeError('Expected bytes')
-else:
-    from urllib.request import urlopen
-    from urllib.parse import urlencode
-    def to_bytes(obj, charset='utf-8', errors='strict'):
-        if obj is None:
-            return None
-        if isinstance(obj, (bytes, bytearray, memoryview)):
-            return bytes(obj)
-        if isinstance(obj, str):
-            return obj.encode(charset, errors)
-        raise TypeError('Expected bytes')
 
 listeners, names, tokens = {}, {}, {}
 
-
 def websocket_send(url, message, hmac_key=None, group='default'):
     sig = hmac_key and hmac.new(to_bytes(hmac_key), to_bytes(message)).hexdigest() or ''
     params = urlencode(
@@ -138,8 +116,8 @@ class PostHandler(tornado.web.RequestHandler):
         if hmac_key and not 'signature' in self.request.arguments:
             self.send_error(401)
         if 'message' in self.request.arguments:
-            message = self.request.arguments['message'][0]
+            message = self.request.arguments['message'][0].decode(encoding='UTF-8')
-            group = self.request.arguments.get('group', ['default'])[0]
+            group = self.request.arguments.get('group', ['default'])[0].decode(encoding='UTF-8')
             print('%s:MESSAGE to %s:%s' % (time.time(), group, message))
             if hmac_key:
                 signature = self.request.arguments['signature'][0]

gluon/custom_import.py

@@ -8,7 +8,7 @@
 Support for smart import syntax for web2py applications
 -------------------------------------------------------
 """
-from gluon._compat import builtin, unicodeT, PY2, to_native
+from gluon._compat import builtin, unicodeT, PY2, to_native, reload
 import os
 import sys
 import threading

gluon/dal.py

@@ -14,6 +14,7 @@ from pydal import DAL as DAL
 from pydal import Field
 from pydal.objects import Row, Rows, Table, Query, Set, Expression
 from pydal import SQLCustomType, geoPoint, geoLine, geoPolygon
+from pydal.migrator import Migrator, InDBMigrator
 from gluon.serializers import custom_json, xml
 from gluon.utils import web2py_uuid
 from gluon import sqlhtml
@@ -31,7 +32,11 @@ def _default_validators(db, field):
     field_type, field_length = field.type, field.length
     requires = []
 
-    if field_type in (('string', 'text', 'password')):
+    if isinstance(field.options, list) and field.requires:
+        requires = validators.IS_IN_SET(field.options, multiple=field_type.startswith('list:'))
+    elif field.regex and not field.requires:
+        requires.append(validators.IS_REGEX(regex))
+    elif field_type in (('string', 'text', 'password')):
         requires.append(validators.IS_LENGTH(field_length))
     elif field_type == 'json':
         requires.append(validators.IS_EMPTY_OR(validators.IS_JSON()))
@@ -49,42 +54,62 @@ def _default_validators(db, field):
         requires.append(validators.IS_TIME())
     elif field_type == 'datetime':
         requires.append(validators.IS_DATETIME())
-    elif db and field_type.startswith('reference') and \
+    elif db and field_type.startswith('reference'):
-            field_type.find('.') < 0 and \
+        if field_type.find('.') < 0 and field_type[10:] in db.tables:
-            field_type[10:] in db.tables:
+            referenced = db[field_type[10:]]
-        referenced = db[field_type[10:]]
+            if hasattr(referenced, '_format') and referenced._format:
-        if hasattr(referenced, '_format') and referenced._format:
+                requires = validators.IS_IN_DB(db, referenced._id,referenced._format)
-            requires = validators.IS_IN_DB(db, referenced._id,
+            else:
-                                           referenced._format)
+                requires = validators.IS_IN_DB(db, referenced._id)
-            if field.unique:
+        elif field_type.find('.') > 0 and field_type[10:].split('.')[0] in db.tables:
-                requires._and = validators.IS_NOT_IN_DB(db, field)
+            table_field = field_type[10:].split('.')
-            if field.tablename == field_type[10:]:
+            table_name=table_field[0]
-                return validators.IS_EMPTY_OR(requires)
+            field_name=table_field[1]
-            return requires
+            referenced = db[table_name]
-    elif db and field_type.startswith('list:reference') and \
+            if hasattr(referenced, '_format') and referenced._format:
-            field_type.find('.') < 0 and \
+                requires = validators.IS_IN_DB(db, referenced[field_name],referenced._format)
-            field_type[15:] in db.tables:
+            else:
-        referenced = db[field_type[15:]]
+                requires = validators.IS_IN_DB(db, referenced[field_name])
-        if hasattr(referenced, '_format') and referenced._format:
+        if field.unique:
-            requires = validators.IS_IN_DB(db, referenced._id,
+            requires._and = validators.IS_NOT_IN_DB(db, field)
-                                           referenced._format, multiple=True)
+        if not field.notnull:
-        else:
+            requires = validators.IS_EMPTY_OR(requires)
-            requires = validators.IS_IN_DB(db, referenced._id,
+        return requires
-                                           multiple=True)
+    elif db and field_type.startswith('list:reference'):
+        if field_type.find('.') < 0 and field_type[15:] in db.tables:
+            referenced = db[field_type[15:]]
+            if hasattr(referenced, '_format') and referenced._format:
+                requires = validators.IS_IN_DB(db, referenced._id,
+                                               referenced._format, multiple=True)
+            else:
+                requires = validators.IS_IN_DB(db, referenced._id,
+                                               multiple=True)
+        elif field_type.find('.') > 0 and field_type[15:].split('.')[0] in db.tables:
+            table_field = field_type[15:].split('.')
+            table_name=table_field[0]
+            field_name=table_field[1]
+            referenced = db[table_name]
+            if hasattr(referenced, '_format') and referenced._format:
+                requires = validators.IS_IN_DB(db, referenced[field_name],
+                                               referenced._format, multiple=True)
+            else:
+                requires = validators.IS_IN_DB(db, referenced[field_name],
+                                               multiple=True)
         if field.unique:
             requires._and = validators.IS_NOT_IN_DB(db, field)
         if not field.notnull:
             requires = validators.IS_EMPTY_OR(requires)
         return requires    
     # does not get here for reference and list:reference
-    if field.unique:
+    if isinstance(requires, list):
-        requires.insert(0, validators.IS_NOT_IN_DB(db, field))
+        if field.unique:
-    excluded_fields = ['string', 'upload', 'text', 'password', 'boolean']
+            requires.insert(0, validators.IS_NOT_IN_DB(db, field))
-    if (field.notnull or field.unique) and field_type not in excluded_fields:
+        excluded_fields = ['string', 'upload', 'text', 'password', 'boolean']
-        requires.insert(0, validators.IS_NOT_EMPTY())
+        if (field.notnull or field.unique) and field_type not in excluded_fields:
-    elif not field.notnull and not field.unique and requires:
+            requires.insert(0, validators.IS_NOT_EMPTY())
-        requires[0] = \
+        elif not field.notnull and not field.unique and requires:
-            validators.IS_EMPTY_OR(requires[0], null='' if field.type in ('string', 'text', 'password') else None)
+            null = null='' if field.type in ('string', 'text', 'password') else None
+            requires[0] = validators.IS_EMPTY_OR(requires[0], null=null)
     return requires
 
 DAL.serializers = {'json': custom_json, 'xml': xml}

gluon/globals.py

@@ -14,7 +14,7 @@ Contains the classes for the global used variables:
 
 """
 from gluon._compat import pickle, StringIO, copyreg, Cookie, urlparse, PY2, iteritems, to_unicode, to_native, \
-    unicodeT, long, hashlib_md5
+    unicodeT, long, hashlib_md5, urllib_quote
 from gluon.storage import Storage, List
 from gluon.streamer import streamer, stream_file_or_304_or_206, DEFAULT_CHUNK_SIZE
 from gluon.contenttype import contenttype
@@ -220,7 +220,12 @@ class Request(Storage):
 
         if is_json:
             try:
-                json_vars = json_parser.load(body)
+                # In Python 3 versions prior to 3.6 load doesn't accept bytes and
+                # bytearray, so we read the body convert to native and use loads
+                # instead of load.
+                # This line can be simplified to json_vars = json_parser.load(body)
+                # if and when we drop support for python versions under 3.6
+                json_vars = json_parser.loads(to_native(body.read())) 
             except:
                 # incoherent request bodies can still be parsed "ad-hoc"
                 json_vars = {}
@@ -331,11 +336,16 @@ class Request(Storage):
         user_agent = session._user_agent
         if user_agent:
             return user_agent
-        user_agent = user_agent_parser.detect(self.env.http_user_agent)
+        http_user_agent = self.env.http_user_agent or ''
+        user_agent = user_agent_parser.detect(http_user_agent)
         for key, value in user_agent.items():
             if isinstance(value, dict):
                 user_agent[key] = Storage(value)
-        user_agent = session._user_agent = Storage(user_agent)
+        user_agent = Storage(user_agent)
+        user_agent.is_mobile = 'Mobile' in http_user_agent
+        user_agent.is_tablet = 'Tablet' in http_user_agent
+        session._user_agent = user_agent 
+        
         return user_agent
 
     def requires_https(self):
@@ -454,7 +464,7 @@ class Response(Storage):
         for meta in iteritems((self.meta or {})):
             k, v = meta
             if isinstance(v, dict):
-                s += '<meta' + ''.join(' %s="%s"' % (xmlescape(key),
+                s += '<meta' + ''.join(' %s="%s"' % (to_native(xmlescape(key)),
                                                      to_native(xmlescape(v[key]))) for key in v) + ' />\n'
             else:
                 s += '<meta name="%s" content="%s" />\n' % (k, to_native(xmlescape(v)))
@@ -468,45 +478,67 @@ class Response(Storage):
         response.cache_includes = (cache_method, time_expire).
         Example: (cache.disk, 60) # caches to disk for 1 minute.
         """
+        app = current.request.application
+
+        # We start by building a files list in which adjacent files internal to
+        # the application are placed in a list inside the files list.
+        #
+        # We will only minify and concat adjacent internal files as there's
+        # no way to know if changing the order with which the files are apppended
+        # will break things since the order matters in both CSS and JS and 
+        # internal files may be interleaved with external ones.
         files = []
-        ext_files = []
+        # For the adjacent list we're going to use storage List to both distinguish
-        has_js = has_css = False
+        # from the regular list and so we can add attributes
+        internal = List()  
+        internal.has_js = False
+        internal.has_css = False
+        done = set() # to remove duplicates
         for item in self.files:
-            if isinstance(item, (list, tuple)):
+            if not isinstance(item, list):
-                ext_files.append(item)
+                if item in done:
+                    continue
+                done.add(item)
+            if isinstance(item, (list, tuple)) or not item.startswith('/' + app): # also consider items in other web2py applications to be external
+                if internal:
+                    files.append(internal)
+                    internal = List()
+                    internal.has_js = False
+                    internal.has_css = False
+                files.append(item)
                 continue
             if extensions and not item.rpartition('.')[2] in extensions:
                 continue
-            if item in files:
+            internal.append(item)
-                continue
             if item.endswith('.js'):
-                has_js = True
+                internal.has_js = True
             if item.endswith('.css'):
-                has_css = True
+                internal.has_css = True            
-            files.append(item)
+        if internal:
+            files.append(internal)
 
-        if have_minify and ((self.optimize_css and has_css) or (self.optimize_js and has_js)):
+        # We're done we can now minify
-            # cache for 5 minutes by default
+        if have_minify:
-            key = hashlib_md5(repr(files)).hexdigest()
+            for i, f in enumerate(files):
-            cache = self.cache_includes or (current.cache.ram, 60 * 5)
+                if isinstance(f, List) and ((self.optimize_css and f.has_css) or (self.optimize_js and f.has_js)):
+                    # cache for 5 minutes by default
+                    key = hashlib_md5(repr(f)).hexdigest()
+                    cache = self.cache_includes or (current.cache.ram, 60 * 5)
+                    def call_minify(files=f):
+                        return List(minify.minify(files,
+                                             URL('static', 'temp'),
+                                             current.request.folder,
+                                             self.optimize_css,
+                                             self.optimize_js))
+                    if cache:
+                        cache_model, time_expire = cache
+                        files[i] = cache_model('response.files.minified/' + key,
+                                            call_minify,
+                                            time_expire)
+                    else:
+                        files[i] = call_minify()
         
-            def call_minify(files=files):
+        def static_map(s, item):
-                return minify.minify(files,
-                                     URL('static', 'temp'),
-                                     current.request.folder,
-                                     self.optimize_css,
-                                     self.optimize_js)
-            if cache:
-                cache_model, time_expire = cache
-                files = cache_model('response.files.minified/' + key,
-                                    call_minify,
-                                    time_expire)
-            else:
-                files = call_minify()
-
-        files.extend(ext_files)
-        s = []
-        for item in files:
             if isinstance(item, str):
                 f = item.lower().split('?')[0]
                 ext = f.rpartition('.')[2]
@@ -526,6 +558,13 @@ class Response(Storage):
                 if tmpl:
                     s.append(tmpl % item[1])
 
+        s = []
+        for item in files:
+            if isinstance(item, List):
+                for f in item:
+                    static_map(s, f)
+            else:
+                static_map(s, item)
         self.write(''.join(s), escape=False)
 
     def stream(self,
@@ -641,6 +680,11 @@ class Response(Storage):
         if download_filename is None:
             download_filename = filename
         if attachment:
+            # Browsers still don't have a simple uniform way to have non ascii
+            # characters in the filename so for now we are percent encoding it
+            if isinstance(download_filename, unicodeT):
+                download_filename = download_filename.encode('utf-8')
+            download_filename = urllib_quote(download_filename)
             headers['Content-Disposition'] = \
                 'attachment; filename="%s"' % download_filename.replace('"', '\"')
         return self.stream(stream, chunk_size=chunk_size, request=request)

gluon/html.py

@@ -1419,20 +1419,21 @@ class LINK(DIV):
 
 class SCRIPT(DIV):
 
-    tag = 'script'
+    tag = b'script'
+    tagname = to_bytes(tag)
 
     def xml(self):
         (fa, co) = self._xml()
-        fa = to_native(fa)
+        fa = to_bytes(fa)
         # no escaping of subcomponents
-        co = '\n'.join([str(component) for component in
+        co = b'\n'.join([to_bytes(component) for component in
                        self.components])
         if co:
             # <script [attributes]><!--//--><![CDATA[//><!--
             # script body
             # //--><!]]></script>
             # return '<%s%s><!--//--><![CDATA[//><!--\n%s\n//--><!]]></%s>' % (self.tag, fa, co, self.tag)
-            return '<%s%s><!--\n%s\n//--></%s>' % (self.tag, fa, co, self.tag)
+            return b'<%s%s><!--\n%s\n//--></%s>' % (self.tagname, fa, co, self.tagname)
         else:
             return DIV.xml(self)
 
@@ -1440,18 +1441,19 @@ class SCRIPT(DIV):
 class STYLE(DIV):
 
     tag = 'style'
+    tagname = to_bytes(tag)
 
     def xml(self):
         (fa, co) = self._xml()
-        fa = to_native(fa)
+        fa = to_bytes(fa)
         # no escaping of subcomponents
-        co = '\n'.join([str(component) for component in
+        co = b'\n'.join([to_bytes(component) for component in
                        self.components])
         if co:
             # <style [attributes]><!--/*--><![CDATA[/*><!--*/
             # style body
             # /*]]>*/--></style>
-            return '<%s%s><!--/*--><![CDATA[/*><!--*/\n%s\n/*]]>*/--></%s>' % (self.tag, fa, co, self.tag)
+            return b'<%s%s><!--/*--><![CDATA[/*><!--*/\n%s\n/*]]>*/--></%s>' % (self.tagname, fa, co, self.tagname)
         else:
             return DIV.xml(self)
 
@@ -2431,7 +2433,7 @@ class BEAUTIFY(DIV):
         if level == 0:
             return
         for c in self.components:
-            if hasattr(c, 'value') and not callable(c.value):
+            if hasattr(c, 'value') and not callable(c.value) and not isinstance(c, cgi.FieldStorage):
                 if c.value:
                     components.append(c.value)
             if hasattr(c, 'xml') and callable(c.xml):

gluon/http.py

@@ -11,7 +11,7 @@ HTTP statuses helpers
 """
 
 import re
-from gluon._compat import iteritems, unicodeT
+from gluon._compat import iteritems, unicodeT, to_bytes
 
 __all__ = ['HTTP', 'redirect']
 
@@ -111,6 +111,8 @@ class HTTP(Exception):
             if not body:
                 body = status
             if isinstance(body, (str, bytes, bytearray)):
+                if isinstance(body, unicodeT):
+                    body = to_bytes(body) # This must be done before len
                 headers['Content-Length'] = len(body)
         rheaders = []
         for k, v in iteritems(headers):
@@ -123,12 +125,15 @@ class HTTP(Exception):
             return ['']
         elif isinstance(body, (str, bytes, bytearray)):
             if isinstance(body, unicodeT):
-                body = body.encode('utf-8')
+                body = to_bytes(body)
             return [body]
         elif hasattr(body, '__iter__'):
             return body
         else:
-            return [str(body)]
+            body = str(body)
+            if isinstance(body, unicodeT):
+                body = to_bytes(body)
+            return [body]
 
     @property
     def message(self):

gluon/languages.py

@@ -18,10 +18,11 @@ import pkgutil
 import logging
 from cgi import escape
 from threading import RLock
-from gluon.utf8 import Utf8
+
 from gluon.utils import local_html_escape
 
 from gluon._compat import copyreg, PY2, maketrans, iterkeys, unicodeT, to_unicode, to_bytes, iteritems, to_native, pjoin
+
 from pydal.contrib.portalocker import read_locked, LockedFile
 
 from gluon.fileutils import listdir
@@ -49,8 +50,10 @@ DEFAULT_CONSTRUCT_PLURAL_FORM = lambda word, plural_id: word
 
 if PY2:
     NUMBERS = (int, long, float)
+    from gluon.utf8 import Utf8
 else:
     NUMBERS = (int, float)
+    Utf8 = str
 
 # pattern to find T(blah blah blah) expressions
 PY_STRING_LITERAL_RE = r'(?<=[^\w]T\()(?P<name>'\
@@ -107,15 +110,17 @@ def markmin(s):
 
 
 def upper_fun(s):
-    return unicode(s, 'utf-8').upper().encode('utf-8')
+    return to_unicode(s).upper()
 
 
 def title_fun(s):
-    return unicode(s, 'utf-8').title().encode('utf-8')
+    return to_unicode(s).title()
 
 
 def cap_fun(s):
-    return unicode(s, 'utf-8').capitalize().encode('utf-8')
+    return to_unicode(s).capitalize()
+
+
 ttab_in = maketrans("\\%{}", '\x1c\x1d\x1e\x1f')
 ttab_out = maketrans('\x1c\x1d\x1e\x1f', "\\%{}")
 
@@ -426,10 +431,16 @@ class lazyT(object):
         return str(self) if self.M else local_html_escape(str(self), quote=False)
 
     def encode(self, *a, **b):
-        return str(self).encode(*a, **b)
+        if PY2 and a[0] != 'utf8':
+            return to_unicode(str(self)).encode(*a, **b)
+        else:
+            return str(self)
 
     def decode(self, *a, **b):
-        return str(self).decode(*a, **b)
+        if PY2:
+            return str(self).decode(*a, **b)
+        else:
+            return str(self)
 
     def read(self):
         return str(self)

gluon/main.py

@@ -183,12 +183,13 @@ def serve_controller(request, response, session):
         response._view_environment.update(page)
         page = run_view_in(response._view_environment)
 
-    # logic to garbage collect after exec, not always, once every 100 requests
+    if not request.env.web2py_disable_garbage_collect:
-    global requests
+        # logic to garbage collect after exec, not always, once every 100 requests
-    requests = ('requests' in globals()) and (requests + 1) % 100 or 0
+        global requests
-    if not requests:
+        requests = ('requests' in globals()) and (requests + 1) % 100 or 0
-        gc.collect()
+        if not requests:
-    # end garbage collection logic
+            gc.collect()
+        # end garbage collection logic
 
     # ##################################################
     # set default headers it not set
@@ -231,7 +232,7 @@ class LazyWSGI(object):
 
         to call third party WSGI applications
         """
-        self.response.status = str(status).split(' ', 1)[0]
+        self.response.status = int(str(status).split(' ', 1)[0])
         self.response.headers = dict(headers)
         return lambda *args, **kargs: \
             self.response.write(escape=False, *args, **kargs)
@@ -745,7 +746,7 @@ class HttpServer(object):
         sock_list = [ip, port]
         if not ssl_certificate or not ssl_private_key:
             logger.info('SSL is off')
-        elif not rocket.ssl:
+        elif not rocket.has_ssl:
             logger.warning('Python "ssl" module unavailable. SSL is OFF')
         elif not exists(ssl_certificate):
             logger.warning('unable to open SSL certificate. SSL is OFF')

gluon/restricted.py

@@ -10,7 +10,7 @@ Restricted environment to execute application's code
 """
 
 import sys
-from gluon._compat import pickle, ClassType
+from gluon._compat import pickle, ClassType, unicodeT, to_bytes
 import traceback
 import types
 import os
@@ -192,10 +192,10 @@ class RestrictedError(Exception):
         # safely show an useful message to the user
         try:
             output = self.output
-            if isinstance(output, unicode):
+            if not isinstance(output, str, bytes, bytearray):
-                output = output.encode("utf8")
-            elif not isinstance(output, str):
                 output = str(output)
+            if isinstance(output, unicodeT):
+                output = to_bytes(output)
         except:
             output = ""
         return output
@@ -205,7 +205,7 @@ def compile2(code, layer):
     return compile(code, layer, 'exec')
 
 
-def restricted(ccode, environment=None, layer='Unknown'):
+def restricted(ccode, environment=None, layer='Unknown', scode=None):
     """
     Runs code in environment and returns the output. If an exception occurs
     in code it raises a RestrictedError containing the traceback. Layer is
@@ -230,7 +230,9 @@ def restricted(ccode, environment=None, layer='Unknown'):
             sys.excepthook(etype, evalue, tb)
         del tb
         output = "%s %s" % (etype, evalue)
-        raise RestrictedError(layer, ccode, output, environment)
+        # Save source code in ticket when available
+        scode = scode if scode else ccode
+        raise RestrictedError(layer, scode, output, environment)
 
 
 def snapshot(info=None, context=5, code=None, environment=None):

gluon/rewrite.py

@@ -22,12 +22,11 @@ import re
 import logging
 import traceback
 import threading
-import urllib
 from gluon.storage import Storage, List
 from gluon.http import HTTP
 from gluon.fileutils import abspath, read_file
 from gluon.settings import global_settings
-from gluon._compat import urllib_unquote, urllib_quote, iteritems, xrange
+from gluon._compat import urllib_unquote, urllib_quote, iteritems, xrange, urllib_quote_plus
 
 isdir = os.path.isdir
 isfile = os.path.isfile
@@ -235,7 +234,7 @@ def try_rewrite_on_error(http_response, request, environ, ticket=None):
                     path_info, query_string = uri, ''
                 query_string += \
                     'code=%s&ticket=%s&requested_uri=%s&request_url=%s' % \
-                    (status, ticket, urllib.quote_plus(
+                    (status, ticket, urllib_quote_plus(
                         request.env.request_uri), request.url)
                 if uri.startswith('http://') or uri.startswith('https://'):
                     # make up a response
@@ -270,12 +269,12 @@ def try_redirect_on_error(http_object, request, ticket=None):
                 elif '?' in redir:
                     url = '%s&code=%s&ticket=%s&requested_uri=%s&request_url=%s' % \
                         (redir, status, ticket,
-                         urllib.quote_plus(request.env.request_uri),
+                         urllib_quote_plus(request.env.request_uri),
                          request.url)
                 else:
                     url = '%s?code=%s&ticket=%s&requested_uri=%s&request_url=%s' % \
                         (redir, status, ticket,
-                         urllib.quote_plus(request.env.request_uri),
+                         urllib_quote_plus(request.env.request_uri),
                          request.url)
                 return HTTP(303, 'You are being redirected <a href="%s">here</a>' % url, Location=url)
     return http_object

gluon/rocket.py

@@ -1750,7 +1750,7 @@ class WSGIWorker(Worker):
         if self.request_method != 'HEAD':
             try:
                 if self.chunked:
-                    self.conn.sendall(b('%x\r\n%s\r\n' % (len(data), data)))
+                    self.conn.sendall(b'%x\r\n%s\r\n' % (len(data), to_bytes(data, 'ISO-8859-1')))
                 else:
                     self.conn.sendall(to_bytes(data))
             except socket.timeout:

gluon/sanitizer.py

@@ -145,7 +145,7 @@ class XssCleaner(HTMLParser):
         if url.startswith('#'):
             return True
         else:
-            parsed = urlparse(url)
+            parsed = urlparse.urlparse(url)
             return ((parsed[0] in self.allowed_schemes and '.' in parsed[1]) or
                     (parsed[0] in self.allowed_schemes and '@' in parsed[2]) or
                     (parsed[0] == '' and parsed[2].startswith('/')))

gluon/scheduler.py

@@ -525,7 +525,7 @@ class MetaScheduler(threading.Thread):
         self.have_heartbeat = True   # set to False to kill
         self.empty_runs = 0
 
-    def async(self, task):
+    def local_async(self, task):
         """Start the background process.
 
         Args:
@@ -913,7 +913,7 @@ class Scheduler(MetaScheduler):
                     self.w_stats.empty_runs = 0
                     self.w_stats.status = RUNNING
                     self.w_stats.total += 1
-                    self.wrapped_report_task(task, self.async(task))
+                    self.wrapped_report_task(task, self.local_async(task))
                     if not self.w_stats.status == DISABLED:
                         self.w_stats.status = ACTIVE
                 else:

gluon/serializers.py

@@ -119,8 +119,8 @@ def xml(value, encoding='UTF-8', key='document', quote=True):
     return ('<?xml version="1.0" encoding="%s"?>' % encoding) + str(xml_rec(value, key, quote))
 
 
-def json(value, default=custom_json, indent=None):
+def json(value, default=custom_json, indent=None, sort_keys=False):
-    value = json_parser.dumps(value, default=default, sort_keys=True, indent=indent)
+    value = json_parser.dumps(value, default=default, sort_keys=sort_keys, indent=indent)
     # replace JavaScript incompatible spacing
     # http://timelessrepo.com/json-isnt-a-javascript-subset
     # PY3 FIXME

gluon/shell.py

@@ -31,10 +31,16 @@ from gluon.globals import Request, Response, Session
 from gluon.storage import Storage, List
 from gluon.admin import w2p_unpack
 from pydal.base import BaseAdapter
-from gluon._compat import iteritems, ClassType
+from gluon._compat import iteritems, ClassType, PY2
 
 logger = logging.getLogger("web2py")
 
+if not PY2:
+    def execfile(filename, global_vars=None, local_vars=None):
+        with open(filename) as f:
+            code = compile(f.read(), filename, 'exec')
+            exec(code, global_vars, local_vars)
+
 
 def enable_autocomplete_and_history(adir, env):
     try:

gluon/sqlhtml.py

@@ -15,11 +15,11 @@ Holds:
 """
 
 import datetime
-import urllib
 import re
+import copy
 
 import os
-from gluon._compat import StringIO, unichr, urllib_quote, iteritems, basestring, long, unicodeT, to_native, to_unicode
+from gluon._compat import StringIO, unichr, urllib_quote, iteritems, basestring, long, unicodeT, to_native, to_unicode, urlencode
 from gluon.http import HTTP, redirect
 from gluon.html import XmlComponent, truncate_string
 from gluon.html import XML, SPAN, TAG, A, DIV, CAT, UL, LI, TEXTAREA, BR, IMG
@@ -658,7 +658,8 @@ class AutocompleteWidget(object):
                  orderby=None, limitby=(0, 10), distinct=False,
                  keyword='_autocomplete_%(tablename)s_%(fieldname)s',
                  min_length=2, help_fields=None, help_string=None,
-                 at_beginning=True, default_var='ac'):
+                 at_beginning=True, default_var='ac', user_signature=True,
+                 hash_vars=False):
 
         self.help_fields = help_fields or []
         self.help_string = help_string
@@ -681,9 +682,10 @@ class AutocompleteWidget(object):
         else:
             self.is_reference = False
         if hasattr(request, 'application'):
-            urlvars = request.vars
+            urlvars = copy.copy(request.vars)
             urlvars[default_var] = 1
-            self.url = URL(args=request.args, vars=urlvars)
+            self.url = URL(args=request.args, vars=urlvars,
+                           user_signature=user_signature, hash_vars=hash_vars)
             self.run_callback = True
         else:
             self.url = request
@@ -1085,6 +1087,106 @@ def formstyle_bootstrap3_inline_factory(col_label_size=3):
         return parent
     return _inner
 
+# bootstrap 4
+def formstyle_bootstrap4_stacked(form, fields):
+    """ bootstrap 3 format form layout
+
+    Note:
+        Experimental!
+    """
+    parent = CAT()
+    for id, label, controls, help in fields:
+        # wrappers
+        _help = SPAN(help, _class='help-block')
+        # embed _help into _controls
+        _controls = CAT(controls, _help)
+        if isinstance(controls, INPUT):
+            if controls['_type'] == 'submit':
+                controls.add_class('btn btn-primary')
+            if controls['_type'] == 'button':
+                controls.add_class('btn btn-secondary')
+            elif controls['_type'] == 'file':
+                controls.add_class('input-file')
+            elif controls['_type'] in ('text', 'password'):
+                controls.add_class('form-control')
+            elif controls['_type'] == 'checkbox':
+                label['_for'] = None
+                label.insert(0, controls)
+                label.insert(0, ' ')
+                _controls = DIV(label, _help, _class="checkbox")
+                label = ''
+            elif isinstance(controls, (SELECT, TEXTAREA)):
+                controls.add_class('form-control')
+
+        elif isinstance(controls, SPAN):
+            _controls = P(controls.components)
+
+        elif isinstance(controls, UL):
+            for e in controls.elements("input"):
+                e.add_class('form-control')
+
+        elif isinstance(controls, CAT) and isinstance(controls[0], INPUT):
+            controls[0].add_class('form-control')
+
+        if isinstance(label, LABEL):
+            label['_class'] = add_class(label.get('_class'), 'form-control-label')
+
+        parent.append(DIV(label, _controls, _class='form-group', _id=id))
+    return parent
+
+
+def formstyle_bootstrap4_inline_factory(col_label_size=3):
+    """ bootstrap 3 horizontal form layout
+
+    Note:
+        Experimental!
+    """
+    def _inner(form, fields):
+        form.add_class('form-horizontal')
+        label_col_class = "col-sm-%d" % col_label_size
+        col_class = "col-sm-%d" % (12 - col_label_size)
+        offset_class = "col-sm-offset-%d" % col_label_size
+        parent = CAT()
+        for id, label, controls, help in fields:
+            # wrappers
+            _help = SPAN(help, _class='help-block')
+            # embed _help into _controls
+            _controls = DIV(controls, _help, _class="%s" % (col_class))
+            if isinstance(controls, INPUT):
+                if controls['_type'] == 'submit':
+                    controls.add_class('btn btn-primary')
+                    _controls = DIV(controls, _class="%s %s" % (col_class, offset_class))
+                if controls['_type'] == 'button':
+                    controls.add_class('btn btn-secondary')
+                elif controls['_type'] == 'file':
+                    controls.add_class('input-file')
+                elif controls['_type'] in ('text', 'password'):
+                    controls.add_class('form-control')
+                elif controls['_type'] == 'checkbox':
+                    label['_for'] = None
+                    label.insert(0, controls)
+                    label.insert(1, ' ')
+                    _controls = DIV(DIV(label, _help, _class="checkbox"),
+                                    _class="%s %s" % (offset_class, col_class))
+                    label = ''
+                elif isinstance(controls, (SELECT, TEXTAREA)):
+                    controls.add_class('form-control')
+
+            elif isinstance(controls, SPAN):
+                _controls = P(controls.components,
+                              _class="form-control-static %s" % col_class)
+            elif isinstance(controls, UL):
+                for e in controls.elements("input"):
+                    e.add_class('form-control')
+            elif isinstance(controls, CAT) and isinstance(controls[0], INPUT):
+                    controls[0].add_class('form-control')
+            if isinstance(label, LABEL):
+                label['_class'] = add_class(label.get('_class'), 'form-control-label %s' % label_col_class)
+
+            parent.append(DIV(label, _controls, _class='form-group row', _id=id))
+        return parent
+    return _inner
+
 
 class SQLFORM(FORM):
 
@@ -1173,6 +1275,8 @@ class SQLFORM(FORM):
         bootstrap=formstyle_bootstrap,
         bootstrap3_stacked=formstyle_bootstrap3_stacked,
         bootstrap3_inline=formstyle_bootstrap3_inline_factory(3),
+        bootstrap4_stacked=formstyle_bootstrap4_stacked,
+        bootstrap4_inline=formstyle_bootstrap4_inline_factory(3),
         inline=formstyle_inline,
         )
 
@@ -1240,9 +1344,16 @@ class SQLFORM(FORM):
         # if no fields are provided, build it from the provided table
         # will only use writable or readable fields, unless forced to ignore
         if fields is None:
-            fields = [f.name for f in table if
+            if not readonly:
-                      (ignore_rw or f.writable or f.readable) and
+                if not record:
-                      (readonly or not f.compute)]
+                    # create form should only show writable fields
+                    fields = [f.name for f in table if (ignore_rw or f.writable) and not f.compute]
+                else:
+                    # update form should also show readable fields and computed fields (but in reaodnly mode)
+                    fields = [f.name for f in table if (ignore_rw or f.writable or f.readable)]
+            else:
+                # read only form should show all readable fields
+                fields = [f.name for f in table if (ignore_rw or f.readable)]
         self.fields = fields
 
         # make sure we have an id
@@ -1325,7 +1436,7 @@ class SQLFORM(FORM):
             label = LABEL(label, label and sep, _for=field_id,
                           _id=field_id + SQLFORM.ID_LABEL_SUFFIX)
 
-            cond = readonly or \
+            cond = readonly or field.compute or \
                 (not ignore_rw and not field.writable and field.readable)
 
             if cond:
@@ -1571,6 +1682,7 @@ class SQLFORM(FORM):
             keepvalues = True if self.record else False
 
         if self.readonly:
+            self.deleted = False
             return False
 
         if request_vars.__class__.__name__ == 'Request':
@@ -1727,7 +1839,7 @@ class SQLFORM(FORM):
                 continue
 
             field = self.table[fieldname]
-            if field.type == 'id':
+            if field.type == 'id' or field.compute:
                 continue
             if field.type == 'boolean':
                 if self.vars.get(fieldname, False):
@@ -1919,8 +2031,10 @@ class SQLFORM(FORM):
         if 'table_name' in attributes:
             del attributes['table_name']
 
-        return SQLFORM(DAL(None).define_table(table_name, *[field.clone() for field in fields]),
+        # Clone fields, while passing tables straight through
-                       **attributes)
+        fields_with_clones = [f.clone() if isinstance(f, Field) else f for f in fields]
+
+        return SQLFORM(DAL(None).define_table(table_name, *fields_with_clones), **attributes)
 
     @staticmethod
     def build_query(fields, keywords):
@@ -1935,6 +2049,8 @@ class SQLFORM(FORM):
             if settings.global_settings.web2py_runtime_gae:
                 return reduce(lambda a,b: a|b, [field.contains(key) for field in sfields])
             else:
+                if not (sfields and key and key.split()):
+                    return fields[0].table
                 return reduce(lambda a,b:a&b,[
                         reduce(lambda a,b: a|b, [
                                 field.contains(k) for field in sfields]
@@ -2159,6 +2275,7 @@ class SQLFORM(FORM):
              represent_none=None,
              showblobs=False):
 
+        dbset = None
         formstyle = formstyle or current.response.formstyle
         if isinstance(query, Set):
             query = query.query
@@ -2190,7 +2307,7 @@ class SQLFORM(FORM):
                       cornerall='',
                       cornertop='',
                       cornerbottom='',
-                      button='button btn btn-default',
+                      button='button btn btn-default btn-secondary',
                       buttontext='buttontext button',
                       buttonadd='icon plus icon-plus glyphicon glyphicon-plus',
                       buttonback='icon leftarrow icon-arrow-left glyphicon glyphicon-arrow-left',
@@ -2340,12 +2457,12 @@ class SQLFORM(FORM):
                 for k, f in iteritems(table):
                     if isinstance(f, Field.Virtual):
                         f.tablename = table._tablename
-            columns = [f for f in fields if f.tablename in tablenames]
+            columns = [f for f in fields if f.tablename in tablenames and f.listable]
         else:
             fields = []
             columns = []
             filter1 = lambda f: isinstance(f, Field) and (f.type!='blob' or showblobs)
-            filter2 = lambda f: isinstance(f, Field) and f.readable
+            filter2 = lambda f: isinstance(f, Field) and f.readable and f.listable
             for table in tables:
                 fields += filter(filter1, table)
                 columns += filter(filter2, table)
@@ -2563,8 +2680,8 @@ class SQLFORM(FORM):
                     try:
                         # the query should be constructed using searchable
                         # fields but not virtual fields
-                        sfields = reduce(lambda a, b: a + b,
+                        is_searchable = lambda f: f.readable and not isinstance(f, Field.Virtual) and f.searchable
-                            [[f for f in t if f.readable and not isinstance(f, Field.Virtual)] for t in tables])
+                        sfields = reduce(lambda a, b: a + b, [filter(is_searchable, t) for t in tables])
                         # use custom_query using searchable
                         if callable(searchable):
                             dbset = dbset(searchable(sfields, keywords))
@@ -2631,8 +2748,8 @@ class SQLFORM(FORM):
                           _id=skeywords_id, _class='form-control',
                           _onfocus="jQuery('#%s').change();jQuery('#%s').slideDown();" % (spanel_id, sfields_id) if advanced_search else ''
                           ),
-                    INPUT(_type='submit', _value=T('Search'), _class="btn btn-default"),
+                    INPUT(_type='submit', _value=T('Search'), _class="btn btn-default btn-secondary"),
-                    INPUT(_type='submit', _value=T('Clear'), _class="btn btn-default",
+                    INPUT(_type='submit', _value=T('Clear'), _class="btn btn-default btn-secondary",
                           _onclick="jQuery('#%s').val('');" % skeywords_id),
                     *hidden_fields,
                     _method="GET", _action=url), search_menu)
@@ -3018,7 +3135,7 @@ class SQLFORM(FORM):
                     order=request.vars.order or '',
                     _export_type=k,
                     keywords=keywords or ''))
-                export_links.append(A(T(label), _href=link, _title=title, _class='btn btn-default'))
+                export_links.append(A(T(label), _href=link, _title=title, _class='btn btn-default btn-secondary'))
             export_menu = \
                 DIV(T('Export:'), _class="w2p_export_menu", *export_links)
         else:
@@ -3037,6 +3154,7 @@ class SQLFORM(FORM):
         res.view_form = view_form
         res.search_form = search_form
         res.rows = rows
+        res.dbset = dbset
         return res
 
     @staticmethod
@@ -3319,7 +3437,7 @@ class SQLTABLE(TABLE):
             return
         REGEX_TABLE_DOT_FIELD = sqlrows.db._adapter.REGEX_TABLE_DOT_FIELD
         fieldmap = dict(zip(sqlrows.colnames, sqlrows.fields))
-        tablemap = dict(((f.tablename, f.table) for f in fieldmap.values()))
+        tablemap = dict(((f.tablename, f.table) if isinstance(f, Field) else (f._table._tablename, f._table) for f in fieldmap.values()))
         for table in tablemap.values():
             pref = table._tablename + '.'
             fieldmap.update(((pref+f.name, f) for f in table._virtual_fields))
@@ -3431,7 +3549,7 @@ class SQLTABLE(TABLE):
                             if ref.find('.') >= 0:
                                 tref, fref = ref.split('.')
                                 if hasattr(sqlrows.db[tref], '_primarykey'):
-                                    href = '%s/%s?%s' % (linkto, tref, urllib.urlencode({fref: r}))
+                                    href = '%s/%s?%s' % (linkto, tref, urlencode({fref: r}))
                         r = A(represent(field, r, record), _href=str(href))
                     elif field.represent:
                         if field not in repr_cache:
@@ -3442,7 +3560,7 @@ class SQLTABLE(TABLE):
                 elif linkto and hasattr(field._table, '_primarykey')\
                         and fieldname in field._table._primarykey:
                     # have to test this with multi-key tables
-                    key = urllib.urlencode(dict([
+                    key = urlencode(dict([
                                 ((tablename in record
                                       and isinstance(record, Row)
                                       and isinstance(record[tablename], Row)) and
@@ -3562,7 +3680,9 @@ class ExportClass(object):
                 if not self.rows.db._adapter.REGEX_TABLE_DOT_FIELD.match(col):
                     row.append(record._extra[col])
                 else:
-                    (t, f) = col.split('.')
+                    # The grid code modifies rows.colnames, adding double quotes
+                    # around the table and field names -- so they must be removed here.
+                    (t, f) = [name.strip('"') for name in col.split('.')]
                     field = self.rows.db[t][f]
                     if isinstance(record.get(t, None), (Row, dict)):
                         value = record[t][f]

gluon/tests/test_authapi.py

@@ -138,6 +138,9 @@ class TestAuthAPI(unittest.TestCase):
         self.assertTrue('new_password2' in result['errors'])
         result = self.auth.change_password(old_password='bart_password', new_password='1234', new_password2='1234')
         self.assertTrue('old_password' in result['errors'])
+        # Test the default 4 min_length is enforced on change password
+        result = self.auth.change_password(old_password='1234', new_password='123', new_password2='123')
+        self.assertTrue('new_password' in result['errors'])
 
     def test_verify_key(self):
         self.auth.settings.registration_requires_verification = True

gluon/tests/test_globals.py

@@ -158,10 +158,10 @@ class testResponse(unittest.TestCase):
         response.files.append(URL('a', 'static', 'css/file.ts'))
         content = return_includes(response)
         self.assertEqual(content,
+                         '<script src="http://maps.google.com/maps/api/js?sensor=false" type="text/javascript"></script>' +
                          '<script src="https://code.jquery.com/jquery-1.11.3.min.js?var=0" type="text/javascript"></script>' +
                          '<link href="/a/static/css/file.css" rel="stylesheet" type="text/css" />' +
-                         '<script src="/a/static/css/file.ts" type="text/typescript"></script>' +
+                         '<script src="/a/static/css/file.ts" type="text/typescript"></script>'
-                         '<script src="http://maps.google.com/maps/api/js?sensor=false" type="text/javascript"></script>'
                          )
 
         response = Response()
@@ -230,3 +230,14 @@ class testResponse(unittest.TestCase):
         current.session._fixup_before_save()
         cookie = str(current.response.cookies)
         self.assertTrue('httponly' not in cookie.lower())
+
+    def test_include_meta(self):
+        response = Response()
+        response.meta[u'web2py'] = 'web2py'
+        response.include_meta()
+        self.assertEqual(response.body.getvalue(), '\n<meta name="web2py" content="web2py" />\n')
+        response = Response()
+        response.meta[u'meta_dict'] = {u'tag_name':'tag_value'}
+        response.include_meta()
+        self.assertEqual(response.body.getvalue(), '\n<meta tag_name="tag_value" />\n')
+

gluon/tests/test_html.py

@@ -340,18 +340,20 @@ class TestBareHelpers(unittest.TestCase):
 
     def test_SCRIPT(self):
         self.assertEqual(SCRIPT('<>', _a='1', _b='2').xml(),
-                         '''<script a="1" b="2"><!--
+                         b'''<script a="1" b="2"><!--
 <>
 //--></script>''')
         self.assertEqual(SCRIPT('<>').xml(),
-                         '''<script><!--
+                         b'''<script><!--
 <>
 //--></script>''')
         self.assertEqual(SCRIPT().xml(), b'<script></script>')
+        self.assertEqual(SCRIPT(';').xml() + DIV().xml(), 
+                         b'<script><!--\n;\n//--></script><div></div>')
 
     def test_STYLE(self):
         self.assertEqual(STYLE('<>', _a='1', _b='2').xml(),
-                         '<style a="1" b="2"><!--/*--><![CDATA[/*><!--*/\n<>\n/*]]>*/--></style>')
+                         b'<style a="1" b="2"><!--/*--><![CDATA[/*><!--*/\n<>\n/*]]>*/--></style>')
         # Try to hit : return DIV.xml(self)
         self.assertEqual(STYLE().xml(), b'<style></style>')
 

gluon/tests/test_languages.py

@@ -12,7 +12,9 @@ import tempfile
 import unittest
 
 from gluon import languages
-from gluon._compat import PY2
+from gluon._compat import PY2, to_unicode, to_bytes
+from gluon.storage import Messages
+from gluon.html import SPAN
 
 MP_WORKING = 0
 try:
@@ -108,6 +110,8 @@ class TestTranslations(unittest.TestCase):
         T.force('it')
         self.assertEqual(str(T('Hello World')),
                          'Salve Mondo')
+        self.assertEqual(to_unicode(T('Hello World')),
+                         'Salve Mondo')
 
 class TestDummyApp(unittest.TestCase):
 
@@ -179,3 +183,43 @@ def index():
         for key in ['hello', 'world', '%s %%{shop}', 'ahoy']:
             self.assertTrue(key in en_dict)
             self.assertTrue(key in pt_dict)
+
+class TestMessages(unittest.TestCase):
+
+    def setUp(self):
+        if os.path.isdir('gluon'):
+            self.langpath = 'applications/welcome/languages'
+        else:
+            self.langpath = os.path.realpath(
+                '../../applications/welcome/languages')
+        self.http_accept_language = 'en'
+
+    def tearDown(self):
+        pass
+
+    def test_decode(self):
+        T = languages.translator(self.langpath, self.http_accept_language)
+        messages = Messages(T)
+        messages.update({'email_sent':'Email sent', 'test': "ä"})
+        self.assertEqual(to_unicode(messages.email_sent, 'utf-8'), 'Email sent')
+
+class TestHTMLTag(unittest.TestCase):
+
+    def setUp(self):
+        if os.path.isdir('gluon'):
+            self.langpath = 'applications/welcome/languages'
+        else:
+            self.langpath = os.path.realpath(
+                '../../applications/welcome/languages')
+        self.http_accept_language = 'en'
+
+    def tearDown(self):
+        pass
+
+    def test_decode(self):
+        T = languages.translator(self.langpath, self.http_accept_language)
+        elem = SPAN(T("Complete"))
+        self.assertEqual(elem.flatten(), "Complete")
+        elem = SPAN(T("Cannot be empty", language="ru"))
+        self.assertEqual(elem.xml(), to_bytes('<span>Пустое значение недопустимо</span>'))
+        self.assertEqual(elem.flatten(), 'Пустое значение недопустимо')

gluon/tests/test_validators.py

@@ -278,8 +278,68 @@ class TestValidators(unittest.TestCase):
         self.assertEqual(rtn, ('jerry', 'oops'))
         rtn = IS_IN_DB(db, 'person.id', '%(name)s', auto_add=True)('jerry')
         self.assertEqual(rtn, (3, None))
+        # Test it works with reference table
+        db.define_table('ref_table',
+                        Field('name'),
+                        Field('person_id', 'reference person')
+                        )
+        ret = db.ref_table.validate_and_insert(name='test reference table')
+        self.assertFalse(list(ret.errors))
+        ret = db.ref_table.validate_and_insert(name='test reference table', person_id=george_id)
+        self.assertFalse(list(ret.errors))
+        rtn = IS_IN_DB(db, 'ref_table.person_id', '%(name)s')(george_id)
+        self.assertEqual(rtn, (george_id, None))
+        # Test it works with reference table.field and keyed table
+        db.define_table('person_keyed',
+                        Field('name'),
+                        primarykey=['name'])
+        db.person_keyed.insert(name='george')
+        db.person_keyed.insert(name='costanza')
+        rtn = IS_IN_DB(db, 'person_keyed.name')('george')
+        self.assertEqual(rtn, ('george', None))
+        db.define_table('ref_table_field',
+                        Field('name'),
+                        Field('person_name', 'reference person_keyed.name')
+                        )
+        ret = db.ref_table_field.validate_and_insert(name='test reference table.field')
+        self.assertFalse(list(ret.errors))
+        ret = db.ref_table_field.validate_and_insert(name='test reference table.field', person_name='george')
+        self.assertFalse(list(ret.errors))
+        vldtr = IS_IN_DB(db, 'ref_table_field.person_name', '%(name)s')
+        vldtr.options()
+        rtn = vldtr('george')
+        self.assertEqual(rtn, ('george', None))
+        # Test it works with list:reference table
+        db.define_table('list_ref_table',
+                        Field('name'),
+                        Field('person_list', 'list:reference person'))
+        ret = db.list_ref_table.validate_and_insert(name='test list:reference table')
+        self.assertFalse(list(ret.errors))
+        ret = db.list_ref_table.validate_and_insert(name='test list:reference table', person_list=[george_id,costanza_id])
+        self.assertFalse(list(ret.errors))
+        vldtr = IS_IN_DB(db, 'list_ref_table.person_list')
+        vldtr.options()
+        rtn = vldtr([george_id,costanza_id])
+        self.assertEqual(rtn, ([george_id,costanza_id], None))
+        # Test it works with list:reference table.field and keyed table
+        #db.define_table('list_ref_table_field',
+        #                Field('name'),
+        #                Field('person_list', 'list:reference person_keyed.name'))
+        #ret = db.list_ref_table_field.validate_and_insert(name='test list:reference table.field')
+        #self.assertFalse(list(ret.errors))
+        #ret = db.list_ref_table_field.validate_and_insert(name='test list:reference table.field', person_list=['george','costanza'])
+        #self.assertFalse(list(ret.errors))
+        #vldtr = IS_IN_DB(db, 'list_ref_table_field.person_list')
+        #vldtr.options()
+        #rtn = vldtr(['george','costanza'])
+        #self.assertEqual(rtn, (['george','costanza'], None))
         db.person.drop()
         db.category.drop()
+        db.person_keyed.drop()
+        db.ref_table.drop()
+        db.ref_table_field.drop()
+        db.list_ref_table.drop()
+        #db.list_ref_table_field.drop()
 
     def test_IS_NOT_IN_DB(self):
         from gluon.dal import DAL, Field
@@ -444,6 +504,8 @@ class TestValidators(unittest.TestCase):
         self.assertEqual(rtn, (None, 'Enter a value'))
         rtn = IS_NOT_EMPTY()('')
         self.assertEqual(rtn, ('', 'Enter a value'))
+        rtn = IS_NOT_EMPTY()(b'')
+        self.assertEqual(rtn, (b'', 'Enter a value'))
         rtn = IS_NOT_EMPTY()('  ')
         self.assertEqual(rtn, ('  ', 'Enter a value'))
         rtn = IS_NOT_EMPTY()(' \n\t')

gluon/tests/test_web.py

@@ -102,12 +102,12 @@ class TestWeb(LiveTest):
                     password='test',
                     _formname='login')
         client.post('user/login', data=data)
-        self.assertTrue('Welcome Homer' in client.text)
+        self.assertTrue('Homer' in client.text)
 
         # check registration and login were successful
         client.get('index')
 
-        self.assertTrue('Welcome Homer' in client.text)
+        self.assertTrue('Homer' in client.text)
 
         client = WebClient('http://127.0.0.1:8000/admin/default/')
         client.post('index', data=dict(password='testpass'))

gluon/tools.py

@@ -12,7 +12,7 @@ Auth, Mail, PluginManager and various utilities
 
 import base64
 from functools import reduce
-from gluon._compat import pickle, thread, urllib2, Cookie, StringIO
+from gluon._compat import pickle, thread, urllib2, Cookie, StringIO, urlencode
 from gluon._compat import configparser, MIMEBase, MIMEMultipart, MIMEText, Header
 from gluon._compat import Encoders, Charset, long, urllib_quote, iteritems
 from gluon._compat import to_bytes, to_native, add_charset
@@ -27,7 +27,6 @@ import time
 import fnmatch
 import traceback
 import smtplib
-import urllib
 import email.utils
 import random
 import hmac
@@ -873,7 +872,7 @@ class Recaptcha(DIV):
                 and len(recaptcha_challenge_field)):
             self.errors['captcha'] = self.error_message
             return False
-        params = urllib.urlencode({
+        params = urlencode({
             'privatekey': private_key,
             'remoteip': remoteip,
             'challenge': recaptcha_challenge_field,
@@ -1026,7 +1025,7 @@ class Recaptcha2(DIV):
         if not recaptcha_response_field:
             self.errors['captcha'] = self.error_message
             return False
-        params = urllib.urlencode({
+        params = urlencode({
             'secret': self.private_key,
             'remoteip': remoteip,
             'response': recaptcha_response_field,
@@ -1320,7 +1319,7 @@ class AuthJWT(object):
         # is the following safe or should we use
         # calendar.timegm(datetime.datetime.utcnow().timetuple())
         # result seem to be the same (seconds since epoch, in UTC)
-        now = time.mktime(datetime.datetime.now().timetuple())
+        now = time.mktime(datetime.datetime.utcnow().timetuple())
         expires = now + self.expiration
         payload = dict(
             hmac_key=session_auth['hmac_key'],
@@ -1332,7 +1331,7 @@ class AuthJWT(object):
         return payload
 
     def refresh_token(self, orig_payload):
-        now = time.mktime(datetime.datetime.now().timetuple())
+        now = time.mktime(datetime.datetime.utcnow().timetuple())
         if self.verify_expiration:
             orig_exp = orig_payload['exp']
             if orig_exp + self.leeway < now:
@@ -1792,6 +1791,7 @@ class Auth(AuthAPI):
                              servicevalidate='serviceValidate',
                              proxyvalidate='proxyValidate',
                              logout='logout'),
+            cas_create_user=True,
             extra_fields={},
             actions_disabled=[],
             controller=controller,
@@ -2284,6 +2284,7 @@ class Auth(AuthAPI):
         If the user doesn't yet exist, then they are created.
         """
         table_user = self.table_user()
+        create_user = self.settings.cas_create_user
         user = None
         checks = []
         # make a guess about who this user is
@@ -2316,6 +2317,11 @@ class Auth(AuthAPI):
                     update_keys[key] = keys[key]
             user.update_record(**update_keys)
         elif checks:
+            if create_user is False:
+                # Remove current open session a send message
+                self.logout(next=None, onlogout=None, log=None)
+                raise HTTP(403, "Forbidden. User need to be created first.")
+
             if 'first_name' not in keys and 'first_name' in table_user.fields:
                 guess = keys.get('email', 'anonymous').split('@')[0]
                 keys['first_name'] = keys.get('username', guess)
@@ -2491,9 +2497,9 @@ class Auth(AuthAPI):
                 success = True
 
         def build_response(body):
-            return '<?xml version="1.0" encoding="UTF-8"?>\n' +\
+            xml_body = to_native(TAG['cas:serviceResponse'](
-                TAG['cas:serviceResponse'](
+                    body, **{'_xmlns:cas': 'http://www.yale.edu/tp/cas'}).xml())
-                    body, **{'_xmlns:cas': 'http://www.yale.edu/tp/cas'}).xml()
+            return '<?xml version="1.0" encoding="UTF-8"?>\n' + xml_body
         if success:
             if version == 1:
                 message = 'yes\n%s' % user[userfield]
@@ -3044,7 +3050,7 @@ class Auth(AuthAPI):
 
         if self.settings.register_verify_password:
             if self.settings.register_fields is None:
-                self.settings.register_fields = [f.name for f in table_user if f.writable]
+                self.settings.register_fields = [f.name for f in table_user if f.writable and not f.compute]
                 k = self.settings.register_fields.index(passfield)
                 self.settings.register_fields.insert(k + 1, "password_two")
             extra_fields = [
@@ -3656,6 +3662,16 @@ class Auth(AuthAPI):
         if not self.is_logged_in():
             redirect(self.settings.login_url,
                      client_side=self.settings.client_side)
+
+        # Go to external link to change the password
+        if self.settings.login_form != self:
+            cas = self.settings.login_form
+            # To prevent error if change_password_url function is not defined in alternate login
+            if hasattr(cas, 'change_password_url'):
+                next = cas.change_password_url(next)
+                if next is not None:
+                    redirect(next)
+
         db = self.db
         table_user = self.table_user()
         s = db(table_user.id == self.user.id)
@@ -3676,7 +3692,8 @@ class Auth(AuthAPI):
             requires = [requires]
         requires = list(filter(lambda t: isinstance(t, CRYPT), requires))
         if requires:
-            requires[0].min_length = 0
+            requires[0] = CRYPT(**requires[0].__dict__) # Copy the existing CRYPT attributes
+            requires[0].min_length = 0 # But do not enforce minimum length for the old password
         form = SQLFORM.factory(
             Field('old_password', 'password', requires=requires,
                   label=self.messages.old_password),
@@ -3760,9 +3777,9 @@ class Auth(AuthAPI):
             if any(f.compute for f in extra_fields):
                 user = table_user[self.user.id]
                 self._update_session_user(user)
+                self.update_groups()
             else:
                 self.user.update(table_user._filter_fields(form.vars))
-
             session.flash = self.messages.profile_updated
             self.log_event(log, self.user)
             callback(onaccept, form)
@@ -4720,7 +4737,7 @@ def fetch(url, data=None, headers=None,
           user_agent='Mozilla/5.0'):
     headers = headers or {}
     if data is not None:
-        data = urllib.urlencode(data)
+        data = urlencode(data)
     if user_agent:
         headers['User-agent'] = user_agent
     headers['Cookie'] = ' '.join(

gluon/utf8.py

@@ -11,7 +11,7 @@ Utilities and class for UTF8 strings managing
 ----------------------------------------------
 """
 from __future__ import print_function
-from gluon._compat import builtin as __builtin__, unicodeT, iteritems, to_unicode, to_native
+from gluon._compat import builtin as __builtin__, unicodeT, iteritems, to_unicode, to_native, reload
 
 __all__ = ['Utf8']
 

gluon/validators.py

@@ -996,7 +996,7 @@ def is_empty(value, empty_regex=None):
         value = value.strip()
         if empty_regex is not None and empty_regex.match(value):
             value = ''
-    if value is None or value == '' or value == []:
+    if value is None or value == '' or value == b'' or value == []:
         return (_value, True)
     return (_value, False)
 
@@ -1656,173 +1656,292 @@ class IS_GENERIC_URL(Validator):
         # else the URL is not valid
         return (value, translate(self.error_message))
 
-# Sources (obtained 2015-Feb-24):
+# Sources (obtained 2017-Nov-11):
 #    http://data.iana.org/TLD/tlds-alpha-by-domain.txt
 # see scripts/parse_top_level_domains.py for an easy update
 
 official_top_level_domains = [
     # a
-    'abogado', 'ac', 'academy', 'accountants', 'active', 'actor',
+    'aaa', 'aarp', 'abarth', 'abb', 'abbott', 'abbvie', 'abc',
-    'ad', 'adult', 'ae', 'aero', 'af', 'ag', 'agency', 'ai',
+    'able', 'abogado', 'abudhabi', 'ac', 'academy', 'accenture',
-    'airforce', 'al', 'allfinanz', 'alsace', 'am', 'amsterdam', 'an',
+    'accountant', 'accountants', 'aco', 'active', 'actor', 'ad',
-    'android', 'ao', 'apartments', 'aq', 'aquarelle', 'ar', 'archi',
+    'adac', 'ads', 'adult', 'ae', 'aeg', 'aero', 'aetna', 'af',
-    'army', 'arpa', 'as', 'asia', 'associates', 'at', 'attorney',
+    'afamilycompany', 'afl', 'africa', 'ag', 'agakhan', 'agency',
-    'au', 'auction', 'audio', 'autos', 'aw', 'ax', 'axa', 'az',
+    'ai', 'aig', 'aigo', 'airbus', 'airforce', 'airtel', 'akdn',
+    'al', 'alfaromeo', 'alibaba', 'alipay', 'allfinanz', 'allstate',
+    'ally', 'alsace', 'alstom', 'am', 'americanexpress',
+    'americanfamily', 'amex', 'amfam', 'amica', 'amsterdam',
+    'analytics', 'android', 'anquan', 'anz', 'ao', 'aol',
+    'apartments', 'app', 'apple', 'aq', 'aquarelle', 'ar', 'arab',
+    'aramco', 'archi', 'army', 'arpa', 'art', 'arte', 'as', 'asda',
+    'asia', 'associates', 'at', 'athleta', 'attorney', 'au',
+    'auction', 'audi', 'audible', 'audio', 'auspost', 'author',
+    'auto', 'autos', 'avianca', 'aw', 'aws', 'ax', 'axa', 'az',
+    'azure',
     # b
-    'ba', 'band', 'bank', 'bar', 'barclaycard', 'barclays',
+    'ba', 'baby', 'baidu', 'banamex', 'bananarepublic', 'band',
-    'bargains', 'bayern', 'bb', 'bd', 'be', 'beer', 'berlin', 'best',
+    'bank', 'bar', 'barcelona', 'barclaycard', 'barclays',
-    'bf', 'bg', 'bh', 'bi', 'bid', 'bike', 'bingo', 'bio', 'biz',
+    'barefoot', 'bargains', 'baseball', 'basketball', 'bauhaus',
-    'bj', 'black', 'blackfriday', 'bloomberg', 'blue', 'bm', 'bmw',
+    'bayern', 'bb', 'bbc', 'bbt', 'bbva', 'bcg', 'bcn', 'bd', 'be',
-    'bn', 'bnpparibas', 'bo', 'boo', 'boutique', 'br', 'brussels',
+    'beats', 'beauty', 'beer', 'bentley', 'berlin', 'best',
-    'bs', 'bt', 'budapest', 'build', 'builders', 'business', 'buzz',
+    'bestbuy', 'bet', 'bf', 'bg', 'bh', 'bharti', 'bi', 'bible',
-    'bv', 'bw', 'by', 'bz', 'bzh',
+    'bid', 'bike', 'bing', 'bingo', 'bio', 'biz', 'bj', 'black',
+    'blackfriday', 'blanco', 'blockbuster', 'blog', 'bloomberg',
+    'blue', 'bm', 'bms', 'bmw', 'bn', 'bnl', 'bnpparibas', 'bo',
+    'boats', 'boehringer', 'bofa', 'bom', 'bond', 'boo', 'book',
+    'booking', 'boots', 'bosch', 'bostik', 'boston', 'bot',
+    'boutique', 'box', 'br', 'bradesco', 'bridgestone', 'broadway',
+    'broker', 'brother', 'brussels', 'bs', 'bt', 'budapest',
+    'bugatti', 'build', 'builders', 'business', 'buy', 'buzz', 'bv',
+    'bw', 'by', 'bz', 'bzh',
     # c
-    'ca', 'cab', 'cal', 'camera', 'camp', 'cancerresearch', 'canon',
+    'ca', 'cab', 'cafe', 'cal', 'call', 'calvinklein', 'cam',
-    'capetown', 'capital', 'caravan', 'cards', 'care', 'career',
+    'camera', 'camp', 'cancerresearch', 'canon', 'capetown',
-    'careers', 'cartier', 'casa', 'cash', 'casino', 'cat',
+    'capital', 'capitalone', 'car', 'caravan', 'cards', 'care',
-    'catering', 'cbn', 'cc', 'cd', 'center', 'ceo', 'cern', 'cf',
+    'career', 'careers', 'cars', 'cartier', 'casa', 'case', 'caseih',
-    'cg', 'ch', 'channel', 'chat', 'cheap', 'christmas', 'chrome',
+    'cash', 'casino', 'cat', 'catering', 'catholic', 'cba', 'cbn',
-    'church', 'ci', 'citic', 'city', 'ck', 'cl', 'claims',
+    'cbre', 'cbs', 'cc', 'cd', 'ceb', 'center', 'ceo', 'cern', 'cf',
-    'cleaning', 'click', 'clinic', 'clothing', 'club', 'cm', 'cn',
+    'cfa', 'cfd', 'cg', 'ch', 'chanel', 'channel', 'chase', 'chat',
-    'co', 'coach', 'codes', 'coffee', 'college', 'cologne', 'com',
+    'cheap', 'chintai', 'christmas', 'chrome', 'chrysler', 'church',
-    'community', 'company', 'computer', 'condos', 'construction',
+    'ci', 'cipriani', 'circle', 'cisco', 'citadel', 'citi', 'citic',
-    'consulting', 'contractors', 'cooking', 'cool', 'coop',
+    'city', 'cityeats', 'ck', 'cl', 'claims', 'cleaning', 'click',
-    'country', 'cr', 'credit', 'creditcard', 'cricket', 'crs',
+    'clinic', 'clinique', 'clothing', 'cloud', 'club', 'clubmed',
-    'cruises', 'cu', 'cuisinella', 'cv', 'cw', 'cx', 'cy', 'cymru',
+    'cm', 'cn', 'co', 'coach', 'codes', 'coffee', 'college',
-    'cz',
+    'cologne', 'com', 'comcast', 'commbank', 'community', 'company',
+    'compare', 'computer', 'comsec', 'condos', 'construction',
+    'consulting', 'contact', 'contractors', 'cooking',
+    'cookingchannel', 'cool', 'coop', 'corsica', 'country', 'coupon',
+    'coupons', 'courses', 'cr', 'credit', 'creditcard',
+    'creditunion', 'cricket', 'crown', 'crs', 'cruise', 'cruises',
+    'csc', 'cu', 'cuisinella', 'cv', 'cw', 'cx', 'cy', 'cymru',
+    'cyou', 'cz',
     # d
-    'dabur', 'dad', 'dance', 'dating', 'day', 'dclk', 'de', 'deals',
+    'dabur', 'dad', 'dance', 'data', 'date', 'dating', 'datsun',
-    'degree', 'delivery', 'democrat', 'dental', 'dentist', 'desi',
+    'day', 'dclk', 'dds', 'de', 'deal', 'dealer', 'deals', 'degree',
-    'design', 'dev', 'diamonds', 'diet', 'digital', 'direct',
+    'delivery', 'dell', 'deloitte', 'delta', 'democrat', 'dental',
-    'directory', 'discount', 'dj', 'dk', 'dm', 'dnp', 'do', 'docs',
+    'dentist', 'desi', 'design', 'dev', 'dhl', 'diamonds', 'diet',
-    'domains', 'doosan', 'durban', 'dvag', 'dz',
+    'digital', 'direct', 'directory', 'discount', 'discover', 'dish',
+    'diy', 'dj', 'dk', 'dm', 'dnp', 'do', 'docs', 'doctor', 'dodge',
+    'dog', 'doha', 'domains', 'dot', 'download', 'drive', 'dtv',
+    'dubai', 'duck', 'dunlop', 'duns', 'dupont', 'durban', 'dvag',
+    'dvr', 'dz',
     # e
-    'eat', 'ec', 'edu', 'education', 'ee', 'eg', 'email', 'emerck',
+    'earth', 'eat', 'ec', 'eco', 'edeka', 'edu', 'education', 'ee',
-    'energy', 'engineer', 'engineering', 'enterprises', 'equipment',
+    'eg', 'email', 'emerck', 'energy', 'engineer', 'engineering',
-    'er', 'es', 'esq', 'estate', 'et', 'eu', 'eurovision', 'eus',
+    'enterprises', 'epost', 'epson', 'equipment', 'er', 'ericsson',
-    'events', 'everbank', 'exchange', 'expert', 'exposed',
+    'erni', 'es', 'esq', 'estate', 'esurance', 'et', 'etisalat',
+    'eu', 'eurovision', 'eus', 'events', 'everbank', 'exchange',
+    'expert', 'exposed', 'express', 'extraspace',
     # f
-    'fail', 'fans', 'farm', 'fashion', 'feedback', 'fi', 'finance',
+    'fage', 'fail', 'fairwinds', 'faith', 'family', 'fan', 'fans',
-    'financial', 'firmdale', 'fish', 'fishing', 'fit', 'fitness',
+    'farm', 'farmers', 'fashion', 'fast', 'fedex', 'feedback',
-    'fj', 'fk', 'flights', 'florist', 'flowers', 'flsmidth', 'fly',
+    'ferrari', 'ferrero', 'fi', 'fiat', 'fidelity', 'fido', 'film',
-    'fm', 'fo', 'foo', 'football', 'forsale', 'foundation', 'fr',
+    'final', 'finance', 'financial', 'fire', 'firestone', 'firmdale',
-    'frl', 'frogans', 'fund', 'furniture', 'futbol',
+    'fish', 'fishing', 'fit', 'fitness', 'fj', 'fk', 'flickr',
+    'flights', 'flir', 'florist', 'flowers', 'fly', 'fm', 'fo',
+    'foo', 'food', 'foodnetwork', 'football', 'ford', 'forex',
+    'forsale', 'forum', 'foundation', 'fox', 'fr', 'free',
+    'fresenius', 'frl', 'frogans', 'frontdoor', 'frontier', 'ftr',
+    'fujitsu', 'fujixerox', 'fun', 'fund', 'furniture', 'futbol',
+    'fyi',
     # g
-    'ga', 'gal', 'gallery', 'garden', 'gb', 'gbiz', 'gd', 'gdn',
+    'ga', 'gal', 'gallery', 'gallo', 'gallup', 'game', 'games',
-    'ge', 'gent', 'gf', 'gg', 'ggee', 'gh', 'gi', 'gift', 'gifts',
+    'gap', 'garden', 'gb', 'gbiz', 'gd', 'gdn', 'ge', 'gea', 'gent',
-    'gives', 'gl', 'glass', 'gle', 'global', 'globo', 'gm', 'gmail',
+    'genting', 'george', 'gf', 'gg', 'ggee', 'gh', 'gi', 'gift',
-    'gmo', 'gmx', 'gn', 'goldpoint', 'goog', 'google', 'gop', 'gov',
+    'gifts', 'gives', 'giving', 'gl', 'glade', 'glass', 'gle',
-    'gp', 'gq', 'gr', 'graphics', 'gratis', 'green', 'gripe', 'gs',
+    'global', 'globo', 'gm', 'gmail', 'gmbh', 'gmo', 'gmx', 'gn',
-    'gt', 'gu', 'guide', 'guitars', 'guru', 'gw', 'gy',
+    'godaddy', 'gold', 'goldpoint', 'golf', 'goo', 'goodhands',
+    'goodyear', 'goog', 'google', 'gop', 'got', 'gov', 'gp', 'gq',
+    'gr', 'grainger', 'graphics', 'gratis', 'green', 'gripe',
+    'grocery', 'group', 'gs', 'gt', 'gu', 'guardian', 'gucci',
+    'guge', 'guide', 'guitars', 'guru', 'gw', 'gy',
     # h
-    'hamburg', 'hangout', 'haus', 'healthcare', 'help', 'here',
+    'hair', 'hamburg', 'hangout', 'haus', 'hbo', 'hdfc', 'hdfcbank',
-    'hermes', 'hiphop', 'hiv', 'hk', 'hm', 'hn', 'holdings',
+    'health', 'healthcare', 'help', 'helsinki', 'here', 'hermes',
-    'holiday', 'homes', 'horse', 'host', 'hosting', 'house', 'how',
+    'hgtv', 'hiphop', 'hisamitsu', 'hitachi', 'hiv', 'hk', 'hkt',
-    'hr', 'ht', 'hu',
+    'hm', 'hn', 'hockey', 'holdings', 'holiday', 'homedepot',
+    'homegoods', 'homes', 'homesense', 'honda', 'honeywell', 'horse',
+    'hospital', 'host', 'hosting', 'hot', 'hoteles', 'hotels',
+    'hotmail', 'house', 'how', 'hr', 'hsbc', 'ht', 'hu', 'hughes',
+    'hyatt', 'hyundai',
     # i
-    'ibm', 'id', 'ie', 'ifm', 'il', 'im', 'immo', 'immobilien', 'in',
+    'ibm', 'icbc', 'ice', 'icu', 'id', 'ie', 'ieee', 'ifm', 'ikano',
-    'industries', 'info', 'ing', 'ink', 'institute', 'insure', 'int',
+    'il', 'im', 'imamat', 'imdb', 'immo', 'immobilien', 'in',
-    'international', 'investments', 'io', 'iq', 'ir', 'irish', 'is',
+    'industries', 'infiniti', 'info', 'ing', 'ink', 'institute',
-    'it', 'iwc',
+    'insurance', 'insure', 'int', 'intel', 'international', 'intuit',
+    'investments', 'io', 'ipiranga', 'iq', 'ir', 'irish', 'is',
+    'iselect', 'ismaili', 'ist', 'istanbul', 'it', 'itau', 'itv',
+    'iveco', 'iwc',
     # j
-    'jcb', 'je', 'jetzt', 'jm', 'jo', 'jobs', 'joburg', 'jp',
+    'jaguar', 'java', 'jcb', 'jcp', 'je', 'jeep', 'jetzt', 'jewelry',
-    'juegos',
+    'jio', 'jlc', 'jll', 'jm', 'jmp', 'jnj', 'jo', 'jobs', 'joburg',
+    'jot', 'joy', 'jp', 'jpmorgan', 'jprs', 'juegos', 'juniper',
     # k
-    'kaufen', 'kddi', 'ke', 'kg', 'kh', 'ki', 'kim', 'kitchen',
+    'kaufen', 'kddi', 'ke', 'kerryhotels', 'kerrylogistics',
-    'kiwi', 'km', 'kn', 'koeln', 'kp', 'kr', 'krd', 'kred', 'kw',
+    'kerryproperties', 'kfh', 'kg', 'kh', 'ki', 'kia', 'kim',
-    'ky', 'kyoto', 'kz',
+    'kinder', 'kindle', 'kitchen', 'kiwi', 'km', 'kn', 'koeln',
+    'komatsu', 'kosher', 'kp', 'kpmg', 'kpn', 'kr', 'krd', 'kred',
+    'kuokgroup', 'kw', 'ky', 'kyoto', 'kz',
     # l
-    'la', 'lacaixa', 'land', 'lat', 'latrobe', 'lawyer', 'lb', 'lc',
+    'la', 'lacaixa', 'ladbrokes', 'lamborghini', 'lamer',
-    'lds', 'lease', 'legal', 'lgbt', 'li', 'lidl', 'life',
+    'lancaster', 'lancia', 'lancome', 'land', 'landrover', 'lanxess',
-    'lighting', 'limited', 'limo', 'link', 'lk', 'loans',
+    'lasalle', 'lat', 'latino', 'latrobe', 'law', 'lawyer', 'lb',
-    'localhost', 'london', 'lotte', 'lotto', 'lr', 'ls', 'lt',
+    'lc', 'lds', 'lease', 'leclerc', 'lefrak', 'legal', 'lego',
-    'ltda', 'lu', 'luxe', 'luxury', 'lv', 'ly',
+    'lexus', 'lgbt', 'li', 'liaison', 'lidl', 'life',
+    'lifeinsurance', 'lifestyle', 'lighting', 'like', 'lilly',
+    'limited', 'limo', 'lincoln', 'linde', 'link', 'lipsy', 'live',
+    'living', 'lixil', 'lk', 'loan', 'loans', 'localhost', 'locker',
+    'locus', 'loft', 'lol', 'london', 'lotte', 'lotto', 'love',
+    'lpl', 'lplfinancial', 'lr', 'ls', 'lt', 'ltd', 'ltda', 'lu',
+    'lundbeck', 'lupin', 'luxe', 'luxury', 'lv', 'ly',
     # m
-    'ma', 'madrid', 'maison', 'management', 'mango', 'market',
+    'ma', 'macys', 'madrid', 'maif', 'maison', 'makeup', 'man',
-    'marketing', 'marriott', 'mc', 'md', 'me', 'media', 'meet',
+    'management', 'mango', 'map', 'market', 'marketing', 'markets',
-    'melbourne', 'meme', 'memorial', 'menu', 'mg', 'mh', 'miami',
+    'marriott', 'marshalls', 'maserati', 'mattel', 'mba', 'mc',
-    'mil', 'mini', 'mk', 'ml', 'mm', 'mn', 'mo', 'mobi', 'moda',
+    'mckinsey', 'md', 'me', 'med', 'media', 'meet', 'melbourne',
-    'moe', 'monash', 'money', 'mormon', 'mortgage', 'moscow',
+    'meme', 'memorial', 'men', 'menu', 'meo', 'merckmsd', 'metlife',
-    'motorcycles', 'mov', 'mp', 'mq', 'mr', 'ms', 'mt', 'mu',
+    'mg', 'mh', 'miami', 'microsoft', 'mil', 'mini', 'mint', 'mit',
-    'museum', 'mv', 'mw', 'mx', 'my', 'mz',
+    'mitsubishi', 'mk', 'ml', 'mlb', 'mls', 'mm', 'mma', 'mn', 'mo',
+    'mobi', 'mobile', 'mobily', 'moda', 'moe', 'moi', 'mom',
+    'monash', 'money', 'monster', 'mopar', 'mormon', 'mortgage',
+    'moscow', 'moto', 'motorcycles', 'mov', 'movie', 'movistar',
+    'mp', 'mq', 'mr', 'ms', 'msd', 'mt', 'mtn', 'mtr', 'mu',
+    'museum', 'mutual', 'mv', 'mw', 'mx', 'my', 'mz',
     # n
-    'na', 'nagoya', 'name', 'navy', 'nc', 'ne', 'net', 'network',
+    'na', 'nab', 'nadex', 'nagoya', 'name', 'nationwide', 'natura',
-    'neustar', 'new', 'nexus', 'nf', 'ng', 'ngo', 'nhk', 'ni',
+    'navy', 'nba', 'nc', 'ne', 'nec', 'net', 'netbank', 'netflix',
-    'nico', 'ninja', 'nl', 'no', 'np', 'nr', 'nra', 'nrw', 'ntt',
+    'network', 'neustar', 'new', 'newholland', 'news', 'next',
-    'nu', 'nyc', 'nz',
+    'nextdirect', 'nexus', 'nf', 'nfl', 'ng', 'ngo', 'nhk', 'ni',
+    'nico', 'nike', 'nikon', 'ninja', 'nissan', 'nissay', 'nl', 'no',
+    'nokia', 'northwesternmutual', 'norton', 'now', 'nowruz',
+    'nowtv', 'np', 'nr', 'nra', 'nrw', 'ntt', 'nu', 'nyc', 'nz',
     # o
-    'okinawa', 'om', 'one', 'ong', 'onl', 'ooo', 'org', 'organic',
+    'obi', 'observer', 'off', 'office', 'okinawa', 'olayan',
-    'osaka', 'otsuka', 'ovh',
+    'olayangroup', 'oldnavy', 'ollo', 'om', 'omega', 'one', 'ong',
+    'onl', 'online', 'onyourside', 'ooo', 'open', 'oracle', 'orange',
+    'org', 'organic', 'origins', 'osaka', 'otsuka', 'ott', 'ovh',
     # p
-    'pa', 'paris', 'partners', 'parts', 'party', 'pe', 'pf', 'pg',
+    'pa', 'page', 'panasonic', 'panerai', 'paris', 'pars',
-    'ph', 'pharmacy', 'photo', 'photography', 'photos', 'physio',
+    'partners', 'parts', 'party', 'passagens', 'pay', 'pccw', 'pe',
-    'pics', 'pictures', 'pink', 'pizza', 'pk', 'pl', 'place',
+    'pet', 'pf', 'pfizer', 'pg', 'ph', 'pharmacy', 'phd', 'philips',
-    'plumbing', 'pm', 'pn', 'pohl', 'poker', 'porn', 'post', 'pr',
+    'phone', 'photo', 'photography', 'photos', 'physio', 'piaget',
-    'praxi', 'press', 'pro', 'prod', 'productions', 'prof',
+    'pics', 'pictet', 'pictures', 'pid', 'pin', 'ping', 'pink',
-    'properties', 'property', 'ps', 'pt', 'pub', 'pw', 'py',
+    'pioneer', 'pizza', 'pk', 'pl', 'place', 'play', 'playstation',
+    'plumbing', 'plus', 'pm', 'pn', 'pnc', 'pohl', 'poker',
+    'politie', 'porn', 'post', 'pr', 'pramerica', 'praxi', 'press',
+    'prime', 'pro', 'prod', 'productions', 'prof', 'progressive',
+    'promo', 'properties', 'property', 'protection', 'pru',
+    'prudential', 'ps', 'pt', 'pub', 'pw', 'pwc', 'py',
     # q
-    'qa', 'qpon', 'quebec',
+    'qa', 'qpon', 'quebec', 'quest', 'qvc',
     # r
-    're', 'realtor', 'recipes', 'red', 'rehab', 'reise', 'reisen',
+    'racing', 'radio', 'raid', 're', 'read', 'realestate', 'realtor',
-    'reit', 'ren', 'rentals', 'repair', 'report', 'republican',
+    'realty', 'recipes', 'red', 'redstone', 'redumbrella', 'rehab',
-    'rest', 'restaurant', 'reviews', 'rich', 'rio', 'rip', 'ro',
+    'reise', 'reisen', 'reit', 'reliance', 'ren', 'rent', 'rentals',
-    'rocks', 'rodeo', 'rs', 'rsvp', 'ru', 'ruhr', 'rw', 'ryukyu',
+    'repair', 'report', 'republican', 'rest', 'restaurant', 'review',
+    'reviews', 'rexroth', 'rich', 'richardli', 'ricoh',
+    'rightathome', 'ril', 'rio', 'rip', 'rmit', 'ro', 'rocher',
+    'rocks', 'rodeo', 'rogers', 'room', 'rs', 'rsvp', 'ru', 'rugby',
+    'ruhr', 'run', 'rw', 'rwe', 'ryukyu',
     # s
-    'sa', 'saarland', 'sale', 'samsung', 'sarl', 'saxo', 'sb', 'sc',
+    'sa', 'saarland', 'safe', 'safety', 'sakura', 'sale', 'salon',
-    'sca', 'scb', 'schmidt', 'school', 'schule', 'schwarz',
+    'samsclub', 'samsung', 'sandvik', 'sandvikcoromant', 'sanofi',
-    'science', 'scot', 'sd', 'se', 'services', 'sew', 'sexy', 'sg',
+    'sap', 'sapo', 'sarl', 'sas', 'save', 'saxo', 'sb', 'sbi', 'sbs',
-    'sh', 'shiksha', 'shoes', 'shriram', 'si', 'singles', 'sj', 'sk',
+    'sc', 'sca', 'scb', 'schaeffler', 'schmidt', 'scholarships',
-    'sky', 'sl', 'sm', 'sn', 'so', 'social', 'software', 'sohu',
+    'school', 'schule', 'schwarz', 'science', 'scjohnson', 'scor',
-    'solar', 'solutions', 'soy', 'space', 'spiegel', 'sr', 'st',
+    'scot', 'sd', 'se', 'search', 'seat', 'secure', 'security',
-    'style', 'su', 'supplies', 'supply', 'support', 'surf',
+    'seek', 'select', 'sener', 'services', 'ses', 'seven', 'sew',
-    'surgery', 'suzuki', 'sv', 'sx', 'sy', 'sydney', 'systems', 'sz',
+    'sex', 'sexy', 'sfr', 'sg', 'sh', 'shangrila', 'sharp', 'shaw',
+    'shell', 'shia', 'shiksha', 'shoes', 'shop', 'shopping',
+    'shouji', 'show', 'showtime', 'shriram', 'si', 'silk', 'sina',
+    'singles', 'site', 'sj', 'sk', 'ski', 'skin', 'sky', 'skype',
+    'sl', 'sling', 'sm', 'smart', 'smile', 'sn', 'sncf', 'so',
+    'soccer', 'social', 'softbank', 'software', 'sohu', 'solar',
+    'solutions', 'song', 'sony', 'soy', 'space', 'spiegel', 'spot',
+    'spreadbetting', 'sr', 'srl', 'srt', 'st', 'stada', 'staples',
+    'star', 'starhub', 'statebank', 'statefarm', 'statoil', 'stc',
+    'stcgroup', 'stockholm', 'storage', 'store', 'stream', 'studio',
+    'study', 'style', 'su', 'sucks', 'supplies', 'supply', 'support',
+    'surf', 'surgery', 'suzuki', 'sv', 'swatch', 'swiftcover',
+    'swiss', 'sx', 'sy', 'sydney', 'symantec', 'systems', 'sz',
     # t
-    'taipei', 'tatar', 'tattoo', 'tax', 'tc', 'td', 'technology',
+    'tab', 'taipei', 'talk', 'taobao', 'target', 'tatamotors',
-    'tel', 'temasek', 'tennis', 'tf', 'tg', 'th', 'tienda', 'tips',
+    'tatar', 'tattoo', 'tax', 'taxi', 'tc', 'tci', 'td', 'tdk',
-    'tires', 'tirol', 'tj', 'tk', 'tl', 'tm', 'tn', 'to', 'today',
+    'team', 'tech', 'technology', 'tel', 'telecity', 'telefonica',
-    'tokyo', 'tools', 'top', 'toshiba', 'town', 'toys', 'tp', 'tr',
+    'temasek', 'tennis', 'teva', 'tf', 'tg', 'th', 'thd', 'theater',
-    'trade', 'training', 'travel', 'trust', 'tt', 'tui', 'tv', 'tw',
+    'theatre', 'tiaa', 'tickets', 'tienda', 'tiffany', 'tips',
-    'tz',
+    'tires', 'tirol', 'tj', 'tjmaxx', 'tjx', 'tk', 'tkmaxx', 'tl',
+    'tm', 'tmall', 'tn', 'to', 'today', 'tokyo', 'tools', 'top',
+    'toray', 'toshiba', 'total', 'tours', 'town', 'toyota', 'toys',
+    'tr', 'trade', 'trading', 'training', 'travel', 'travelchannel',
+    'travelers', 'travelersinsurance', 'trust', 'trv', 'tt', 'tube',
+    'tui', 'tunes', 'tushu', 'tv', 'tvs', 'tw', 'tz',
     # u
-    'ua', 'ug', 'uk', 'university', 'uno', 'uol', 'us', 'uy', 'uz',
+    'ua', 'ubank', 'ubs', 'uconnect', 'ug', 'uk', 'unicom',
+    'university', 'uno', 'uol', 'ups', 'us', 'uy', 'uz',
     # v
-    'va', 'vacations', 'vc', 've', 'vegas', 'ventures',
+    'va', 'vacations', 'vana', 'vanguard', 'vc', 've', 'vegas',
-    'versicherung', 'vet', 'vg', 'vi', 'viajes', 'video', 'villas',
+    'ventures', 'verisign', 'versicherung', 'vet', 'vg', 'vi',
-    'vision', 'vlaanderen', 'vn', 'vodka', 'vote', 'voting', 'voto',
+    'viajes', 'video', 'vig', 'viking', 'villas', 'vin', 'vip',
-    'voyage', 'vu',
+    'virgin', 'visa', 'vision', 'vista', 'vistaprint', 'viva',
+    'vivo', 'vlaanderen', 'vn', 'vodka', 'volkswagen', 'volvo',
+    'vote', 'voting', 'voto', 'voyage', 'vu', 'vuelos',
     # w
-    'wales', 'wang', 'watch', 'webcam', 'website', 'wed', 'wedding',
+    'wales', 'walmart', 'walter', 'wang', 'wanggou', 'warman',
-    'wf', 'whoswho', 'wien', 'wiki', 'williamhill', 'wme', 'work',
+    'watch', 'watches', 'weather', 'weatherchannel', 'webcam',
-    'works', 'world', 'ws', 'wtc', 'wtf',
+    'weber', 'website', 'wed', 'wedding', 'weibo', 'weir', 'wf',
+    'whoswho', 'wien', 'wiki', 'williamhill', 'win', 'windows',
+    'wine', 'winners', 'wme', 'wolterskluwer', 'woodside', 'work',
+    'works', 'world', 'wow', 'ws', 'wtc', 'wtf',
     # x
-    'xn--1qqw23a', 'xn--3bst00m', 'xn--3ds443g', 'xn--3e0b707e',
+    'xbox', 'xerox', 'xfinity', 'xihuan', 'xin', 'xn--11b4c3d',
-    'xn--45brj9c', 'xn--45q11c', 'xn--4gbrim', 'xn--55qw42g',
+    'xn--1ck2e1b', 'xn--1qqw23a', 'xn--2scrj9c', 'xn--30rr7y',
-    'xn--55qx5d', 'xn--6frz82g', 'xn--6qq986b3xl', 'xn--80adxhks',
+    'xn--3bst00m', 'xn--3ds443g', 'xn--3e0b707e', 'xn--3hcrj9c',
-    'xn--80ao21a', 'xn--80asehdb', 'xn--80aswg', 'xn--90a3ac',
+    'xn--3oq18vl8pn36a', 'xn--3pxu8k', 'xn--42c2d9a', 'xn--45br5cyl',
-    'xn--90ais', 'xn--b4w605ferd', 'xn--c1avg', 'xn--cg4bki',
+    'xn--45brj9c', 'xn--45q11c', 'xn--4gbrim', 'xn--54b7fta0cc',
-    'xn--clchc0ea0b2g2a9gcd', 'xn--czr694b', 'xn--czrs0t',
+    'xn--55qw42g', 'xn--55qx5d', 'xn--5su34j936bgsg', 'xn--5tzm5g',
-    'xn--czru2d', 'xn--d1acj3b', 'xn--d1alf', 'xn--fiq228c5hs',
+    'xn--6frz82g', 'xn--6qq986b3xl', 'xn--80adxhks', 'xn--80ao21a',
-    'xn--fiq64b', 'xn--fiqs8s', 'xn--fiqz9s', 'xn--flw351e',
+    'xn--80aqecdr1a', 'xn--80asehdb', 'xn--80aswg', 'xn--8y0a063a',
-    'xn--fpcrj9c3d', 'xn--fzc2c9e2c', 'xn--gecrj9c', 'xn--h2brj9c',
+    'xn--90a3ac', 'xn--90ae', 'xn--90ais', 'xn--9dbq2a',
-    'xn--hxt814e', 'xn--i1b6b1a6a2e', 'xn--io0a7i', 'xn--j1amh',
+    'xn--9et52u', 'xn--9krt00a', 'xn--b4w605ferd',
-    'xn--j6w193g', 'xn--kprw13d', 'xn--kpry57d', 'xn--kput3i',
+    'xn--bck1b9a5dre4c', 'xn--c1avg', 'xn--c2br7g', 'xn--cck2b3b',
-    'xn--l1acc', 'xn--lgbbat1ad8j', 'xn--mgb9awbf',
+    'xn--cg4bki', 'xn--clchc0ea0b2g2a9gcd', 'xn--czr694b',
-    'xn--mgba3a4f16a', 'xn--mgbaam7a8h', 'xn--mgbab2bd',
+    'xn--czrs0t', 'xn--czru2d', 'xn--d1acj3b', 'xn--d1alf',
-    'xn--mgbayh7gpa', 'xn--mgbbh1a71e', 'xn--mgbc0a9azcg',
+    'xn--e1a4c', 'xn--eckvdtc9d', 'xn--efvy88h', 'xn--estv75g',
-    'xn--mgberp4a5d4ar', 'xn--mgbx4cd0ab', 'xn--ngbc5azd',
+    'xn--fct429k', 'xn--fhbei', 'xn--fiq228c5hs', 'xn--fiq64b',
-    'xn--node', 'xn--nqv7f', 'xn--nqv7fs00ema', 'xn--o3cw4h',
+    'xn--fiqs8s', 'xn--fiqz9s', 'xn--fjq720a', 'xn--flw351e',
-    'xn--ogbpf8fl', 'xn--p1acf', 'xn--p1ai', 'xn--pgbs0dh',
+    'xn--fpcrj9c3d', 'xn--fzc2c9e2c', 'xn--fzys8d69uvgm',
-    'xn--q9jyb4c', 'xn--qcka1pmc', 'xn--rhqv96g', 'xn--s9brj9c',
+    'xn--g2xx48c', 'xn--gckr3f0f', 'xn--gecrj9c', 'xn--gk3at1e',
-    'xn--ses554g', 'xn--unup4y', 'xn--vermgensberater-ctb',
+    'xn--h2breg3eve', 'xn--h2brj9c', 'xn--h2brj9c8c', 'xn--hxt814e',
-    'xn--vermgensberatung-pwb', 'xn--vhquv', 'xn--wgbh1c',
+    'xn--i1b6b1a6a2e', 'xn--imr513n', 'xn--io0a7i', 'xn--j1aef',
+    'xn--j1amh', 'xn--j6w193g', 'xn--jlq61u9w7b', 'xn--jvr189m',
+    'xn--kcrx77d1x4a', 'xn--kprw13d', 'xn--kpry57d', 'xn--kpu716f',
+    'xn--kput3i', 'xn--l1acc', 'xn--lgbbat1ad8j', 'xn--mgb9awbf',
+    'xn--mgba3a3ejt', 'xn--mgba3a4f16a', 'xn--mgba7c0bbn0a',
+    'xn--mgbaakc7dvf', 'xn--mgbaam7a8h', 'xn--mgbab2bd',
+    'xn--mgbai9azgqp6j', 'xn--mgbayh7gpa', 'xn--mgbb9fbpob',
+    'xn--mgbbh1a', 'xn--mgbbh1a71e', 'xn--mgbc0a9azcg',
+    'xn--mgbca7dzdo', 'xn--mgberp4a5d4ar', 'xn--mgbgu82a',
+    'xn--mgbi4ecexp', 'xn--mgbpl2fh', 'xn--mgbt3dhd', 'xn--mgbtx2b',
+    'xn--mgbx4cd0ab', 'xn--mix891f', 'xn--mk1bu44c', 'xn--mxtq1m',
+    'xn--ngbc5azd', 'xn--ngbe9e0a', 'xn--ngbrx', 'xn--node',
+    'xn--nqv7f', 'xn--nqv7fs00ema', 'xn--nyqy26a', 'xn--o3cw4h',
+    'xn--ogbpf8fl', 'xn--p1acf', 'xn--p1ai', 'xn--pbt977c',
+    'xn--pgbs0dh', 'xn--pssy2u', 'xn--q9jyb4c', 'xn--qcka1pmc',
+    'xn--qxam', 'xn--rhqv96g', 'xn--rovu88b', 'xn--rvc1e0am3e',
+    'xn--s9brj9c', 'xn--ses554g', 'xn--t60b56a', 'xn--tckwe',
+    'xn--tiq49xqyj', 'xn--unup4y', 'xn--vermgensberater-ctb',
+    'xn--vermgensberatung-pwb', 'xn--vhquv', 'xn--vuq861b',
+    'xn--w4r85el8fhu5dnra', 'xn--w4rs40l', 'xn--wgbh1c',
     'xn--wgbl6a', 'xn--xhq521b', 'xn--xkc2al3hye2a',
-    'xn--xkc2dl3a5ee0h', 'xn--yfro4i67o', 'xn--ygbi2ammx',
+    'xn--xkc2dl3a5ee0h', 'xn--y9a3aq', 'xn--yfro4i67o',
-    'xn--zfr164b', 'xxx', 'xyz',
+    'xn--ygbi2ammx', 'xn--zfr164b', 'xperia', 'xxx', 'xyz',
     # y
-    'yachts', 'yandex', 'ye', 'yodobashi', 'yoga', 'yokohama',
+    'yachts', 'yahoo', 'yamaxun', 'yandex', 'ye', 'yodobashi',
-    'youtube', 'yt',
+    'yoga', 'yokohama', 'you', 'youtube', 'yt', 'yun',
     # z
-    'za', 'zip', 'zm', 'zone', 'zuerich', 'zw'
+    'za', 'zappos', 'zara', 'zero', 'zip', 'zippo', 'zm', 'zone',
+    'zuerich', 'zw'
 ]
 
 

gluon/widget.py

@@ -1076,7 +1076,10 @@ def start_schedulers(options):
         return
 
     # Work around OS X problem: http://bugs.python.org/issue9405
-    import urllib
+    if PY2:
+        import urllib
+    else:
+        import urllib.request as urllib
     urllib.getproxies()
 
     for app in apps:

scripts/access.wsgi

@@ -35,11 +35,17 @@
 #
 
 URL_CHECK_ACCESS = 'http://127.0.0.1:8002/%(app)s/default/check_access'
+PY2 = sys.version_info[0] == 2
 
 def allow_access(environ,host):
+    if PY2:
+        import urllib2
+        from urllib import urlencode
+    else:
+        from urllib import request as urllib2
+        from urllib.parse import urlencode
+
     import os
-    import urllib
-    import urllib2
     import datetime
     header = '%s @ %s ' % (datetime.datetime.now(),host) + '='*20
     pprint = '\n'.join('%s:%s' % item for item in environ.items())
@@ -56,7 +62,7 @@ def allow_access(environ,host):
         if key.startswith('HTTP_'):
             headers[key[5:]] = environ[key] # this passes the cookies through!
     try:
-        data = urllib.urlencode({'request_uri':environ['REQUEST_URI']})
+        data = urlencode({'request_uri':environ['REQUEST_URI']})
         request = urllib2.Request(URL_CHECK_ACCESS % dict(app=app),data,headers)
         response = urllib2.urlopen(request).read().strip().lower()
         if response.startswith('true'): return True

scripts/check_lang_progress.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Written by Vinyl Darkscratch, www.queengoob.org
+
+import sys
+import os
+
+# TODO: add comments
+
+# This script can be run with no arguments (which sets the language folder to the current working directory, and default language to English), one argument (which sets the default language), or two arguments (language folder path and default language).
+# When run, this script will compare the original language's strings to their assigned values and determine if they match, just like web2py's interface.  While some phrases may be the exact same between both languages, this should provide a good idea of how far along translations are.
+
+def check_lang_progress(cwd, default_lang):
+	for x in os.listdir(cwd):
+		if x == default_lang or x.startswith("plural-"): continue
+		
+		data = eval(open(os.path.join(cwd, x)).read())
+
+		total = 0
+		translated = 0
+
+		for key in data:
+			total += 1
+			if key.replace('@markmin\x01', '') != data[key]: translated += 1
+
+		print "Translations for %s (%s): %d/%d Translated (%d Untranslated)" %(data['!langname!'], data['!langcode!'], translated, total, total-translated)
+
+if __name__ == "__main__":
+	cwd = os.getcwd()
+	default_lang = 'en'
+
+	if len(sys.argv) > 2:
+		cwd = sys.argv[1]
+		default_lang = sys.argv[2]
+	elif len(sys.argv) > 1:
+		default_lang = sys.argv[1]
+
+	check_lang_progress(cwd, default_lang)

scripts/setup-web2py-nginx-uwsgi-ubuntu.sh

@@ -222,7 +222,7 @@ echo <<EOF
 you can stop uwsgi and nginx with
 
   sudo /etc/init.d/nginx stop
-  sudo systemctl start emperor.uwsgi.service
+  sudo systemctl stop emperor.uwsgi.service
  
 and start it with
 

scripts/tickets2db.py

@@ -31,7 +31,7 @@ hashes = {}
 
 while 1:
     if request.tickets_db:
-        print "You're storing tickets yet in database"
+        print("You're storing tickets yet in database")
         sys.exit(1)
 
     for file in os.listdir(errors_path):

scripts/tickets2slack.py

@@ -22,7 +22,7 @@ import json
 try:
     import requests
 except ImportError as e:
-    print "missing module 'Requests', aborting."
+    print("missing module 'Requests', aborting.")
     sys.exit(1)
 
 from gluon import URL

scripts/update_languages.py

@@ -0,0 +1,82 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Written by Vinyl Darkscratch, www.queengoob.org
+
+# TODO: add comments
+
+import os
+import ast
+import sys
+import inspect
+
+currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
+parentdir = os.path.dirname(currentdir)
+sys.path.insert(0, parentdir)
+
+from gluon.cfs import getcfs
+from gluon.utf8 import Utf8
+from gluon._compat import copyreg, PY2, maketrans, iterkeys, unicodeT, to_unicode, to_bytes, iteritems, to_native, pjoin
+from gluon.languages import findT
+
+# This script can be run with no arguments (which sets the application folder to the current working directory, and default language to English), one argument (which sets the default language), or two arguments (application folder path and default language).
+# When run, it will update the default language, as well as strip all of the strings found in the non-default languages but not in the default language, and add the strings found in the default language to the non-default languages it is not, making sure translators don't do additional work that will never be used.
+
+def read_dict_aux(filename):
+	lang_text = open(filename, 'r').read().replace(b'\r\n', b'\n')
+	try:
+		return safe_eval(to_native(lang_text)) or {}
+	except Exception:
+		e = sys.exc_info()[1]
+		status = 'Syntax error in %s (%s)' % (filename, e)
+		return {'__corrupted__': status}
+
+def read_dict(filename):
+	return getcfs('lang:' + filename, filename, lambda: read_dict_aux(filename))
+
+def safe_eval(text):
+	if text.strip():
+		try:
+			return ast.literal_eval(text)
+		except ImportError:
+			return eval(text, {}, {})
+	return None
+
+def sort_function(x, y):
+	return cmp(unicode(x, 'utf-8').lower(), unicode(y, 'utf-8').lower())
+
+def write_file(file, contents):
+	file.write('# -*- coding: utf-8 -*-\n{\n')
+	for key in sorted(contents, sort_function):
+		file.write('%s: %s,\n' % (repr(Utf8(key)),
+								repr(Utf8(contents[key]))))
+	file.write('}\n')
+	file.close()
+
+def update_languages(cwd, default_lang):
+	defaultfp = os.path.join(cwd, "languages", '%s.py' %default_lang)
+	findT(cwd, default_lang)
+	default = read_dict(defaultfp)
+
+	for lang in os.listdir(os.path.join(cwd, "languages")):
+		if lang == default_lang+".py" or lang.startswith("plural-"): continue
+		
+		i18n = read_dict(os.path.join(cwd, "languages", lang))
+		if i18n:
+			new_dict = default
+			for phrase in i18n:
+				if phrase in default:
+					new_dict[phrase] = i18n[phrase]
+			write_file(open(os.path.join(cwd, "languages", lang), 'w'), new_dict)
+			print lang
+
+if __name__ == "__main__":
+	cwd = os.getcwd()
+	default_lang = 'en'
+
+	if len(sys.argv) > 2:
+		cwd = sys.argv[1]
+		default_lang = sys.argv[2]
+	elif len(sys.argv) > 1:
+		default_lang = sys.argv[1]
+
+	update_languages(cwd, default_lang)

scripts/zip_static_files.py

@@ -14,7 +14,7 @@ def zip_static(filelist=[]):
         extension = os.path.splitext(fi)
         extension = len(extension) > 1 and extension[1] or None
         if not extension or extension not in ALLOWED_EXTS:
-            print 'skipping %s' % os.path.basename(fi)
+            print('skipping %s' % os.path.basename(fi))
             continue
         fstats = os.stat(fi)
         atime, mtime = fstats.st_atime, fstats.st_mtime
@@ -23,10 +23,10 @@ def zip_static(filelist=[]):
             zstats = os.stat(gfi)
             zatime, zmtime = zstats.st_atime, zstats.st_mtime
             if zatime == atime and zmtime == mtime:
-                print 'skipping %s, already gzipped to the latest version' % os.path.basename(fi)
+                print('skipping %s, already gzipped to the latest version' % os.path.basename(fi))
                 continue
-        print 'gzipping %s to %s' % (
+        print('gzipping %s to %s' % (
-            os.path.basename(fi), os.path.basename(gfi))
+            os.path.basename(fi), os.path.basename(gfi)))
         f_in = open(fi, 'rb')
         f_out = gzip.open(gfi, 'wb')
         f_out.writelines(f_in)
@@ -36,7 +36,7 @@ def zip_static(filelist=[]):
         saved = fstats.st_size - os.stat(gfi).st_size
         tsave += saved
 
-    print 'saved %s KB' % (int(tsave) / 1000.0)
+    print('saved %s KB' % (int(tsave) / 1000.0))
 
 if __name__ == '__main__':
     ALLOWED_EXTS = ['.css', '.js']

setup.py

@@ -65,6 +65,9 @@ def start():
                     'gluon/contrib/pysimplesoap',
                     'gluon/contrib/pg8000',
                     'gluon/contrib/plural_rules',
+                    'gluon/contrib/minify',
+                    'gluon/contrib/pyaes',
+                    'gluon/contrib/pyuca',
                     'gluon/tests',
                     ],
           package_data={'gluon': ['env.tar']},