R-2.15.4

Minor error in EOF instructions

.travis.yml

@@ -4,11 +4,18 @@ sudo: required
 
 cache: pip
 
+dist: "trusty"
+
 python:
   - '2.7'
-  - 'pypy'
   - '3.5'
   - '3.6'
+  - 'pypy-5.3.1'
+  - 'pypy3.5-5.7.1-beta'
+
+matrix:
+  allow_failures:
+    - python: 'pypy3.5-5.7.1-beta'
 
 install:
   - pip install -e .
@@ -33,3 +40,6 @@ notifications:
 
 addons:
   postgresql: "9.4"
+  apt:
+    packages:
+      - postgresql-9.4-postgis-2.3

CHANGELOG

@@ -1,7 +1,12 @@
-## 2.16.0b1
+## 2.15.1-4
+- pydal 17.08
+- dropped support for python 2.6
+- dropped web shell
 - experimental python 3 support
 - experimental authapi for service login
+- allow ajax file uploads
 - more tests
+- more pep8 compliance
 - d3.js model visulization
 - improved scheduler
 - is_email support for internationalized Domain Names
@@ -20,6 +25,13 @@
 - Updated fpdf to latest version
 - JWT support
 - import fabfile for remote deployment
+- scheduler new feature: you can now specify intervals with cron
+- gluon/* removed from sys.path. Applications relying on statements like e.g.
+  "from storage import Storage"
+  will need to be rewritten with
+  "from gluon.storage import Storage"
+- tests can only be run with the usual web2py.py --run_system_tests OR with
+  python -m unittest -v gluon.tests on the root dir
 - jQuery 3.2.1
 - PyDAL 17.07 including:
   allow jsonb support for postgres
@@ -30,21 +42,8 @@
   improved Teradata support
   improved mongodb support
   overall refactoring
-  experimental support for Google Cloud SQL v2 (TODO)
-
-## 2.15.x
-- web2py does not support python 2.6 anymore
-- py3.5 syntax compatible (see #1353 for details)
-- dropped web shell from admin
-- scheduler new feature: you can now specify intervals with cron
-- gluon/* removed from sys.path. Applications relying on statements like e.g.
-  "from storage import Storage"
-  will need to be rewritten with
-  "from gluon.storage import Storage"
-- tests can only be run with the usual web2py.py --run_system_tests OR with
-  python -m unittest -v gluon.tests on the root dir
-- updated pymysql driver
-
+  experimental support for Google Cloud SQL v2
+  new pymysql driver
 
 ## 2.14.6
 

Makefile

@@ -32,7 +32,7 @@ update:
 	echo "remember that pymysql was tweaked"
 src:
 	### Use semantic versioning
-	echo 'Version 2.14.6-stable+timestamp.'`date +%Y.%m.%d.%H.%M.%S` > VERSION
+	echo 'Version 2.15.4-stable+timestamp.'`date +%Y.%m.%d.%H.%M.%S` > VERSION
 	### rm -f all junk files
 	make clean
 	### clean up baisc apps
@@ -54,7 +54,7 @@ src:
 	### build web2py_src.zip
 	echo '' > NEWINSTALL
 	mv web2py_src.zip web2py_src_old.zip | echo 'no old'
-	cd ..; zip -r web2py/web2py_src.zip web2py/web2py.py web2py/anyserver.py web2py/gluon/* web2py/extras/* web2py/handlers/* web2py/examples/* web2py/README.markdown  web2py/LICENSE web2py/CHANGELOG web2py/NEWINSTALL web2py/VERSION web2py/MANIFEST.in web2py/scripts/*.sh web2py/scripts/*.py web2py/applications/admin web2py/applications/examples/ web2py/applications/welcome web2py/applications/__init__.py web2py/site-packages/__init__.py web2py/gluon/tests/*.sh web2py/gluon/tests/*.py
+	cd ..; zip -r web2py/web2py_src.zip web2py/web2py.py web2py/anyserver.py web2py/fabfile.py web2py/gluon/* web2py/extras/* web2py/handlers/* web2py/examples/* web2py/README.markdown  web2py/LICENSE web2py/CHANGELOG web2py/NEWINSTALL web2py/VERSION web2py/MANIFEST.in web2py/scripts/*.sh web2py/scripts/*.py web2py/applications/admin web2py/applications/examples/ web2py/applications/welcome web2py/applications/__init__.py web2py/site-packages/__init__.py web2py/gluon/tests/*.sh web2py/gluon/tests/*.py
 
 mdp:
 	make src
@@ -97,6 +97,8 @@ win:
 	cp -r applications/welcome ../web2py_win/web2py/applications
 	cp -r applications/examples ../web2py_win/web2py/applications
 	cp applications/__init__.py ../web2py_win/web2py/applications
+	# per https://github.com/web2py/web2py/issues/1716
+	mv ../web2py_win/web2py/_ssl.pyd ../web2py_win/web2py/_ssl.pyd.legacy 
 	cd ../web2py_win; zip -r web2py_win.zip web2py
 	mv ../web2py_win/web2py_win.zip .
 run:

VERSION

@@ -1 +1 @@
-Version 2.14.6-stable+timestamp.2016.05.09.19.18.48
+Version 2.15.4-stable+timestamp.2017.09.01.22.38.25

applications/admin/controllers/appadmin.py

@@ -657,39 +657,38 @@ def d3_graph_model():
     Create a list of table dicts, called "nodes"
     """
     
-    data = {}
     nodes = []
     links = []
 
-    subgraphs = dict()
+    for database in databases:
+        db = eval_in_global_env(database)
+        for tablename in db.tables:
+            fields = []
+            for field in db[tablename]:
+                f_type = field.type
+                if not isinstance(f_type,str):
+                    disp = ' '
+                elif f_type == 'string':
+                    disp =  field.length
+                elif f_type == 'id':
+                    disp =  "PK"
+                elif f_type.startswith('reference') or \
+                    f_type.startswith('list:reference'):
+                    disp = "FK"
+                else:
+                    disp = ' '
+                fields.append(dict(name= field.name, type=field.type, disp = disp))
 
-    for tablename in db.tables:
-        fields = []
-        for field in db[tablename]:
-            f_type = field.type
-            if not isinstance(f_type,str):
-                disp = ' '
-            elif f_type == 'string':
-                disp =  field.length
-            elif f_type == 'id':
-                disp =  "PK"
-            elif f_type.startswith('reference') or \
-                f_type.startswith('list:reference'):
-                disp = "FK"
-            else:
-                disp = ' '
-            fields.append(dict(name= field.name, type=field.type, disp = disp))
+                if isinstance(f_type,str) and (
+                    f_type.startswith('reference') or
+                    f_type.startswith('list:reference')):
+                    referenced_table = f_type.split()[1].split('.')[0]
 
-            if isinstance(f_type,str) and (
-                f_type.startswith('reference') or
-                f_type.startswith('list:reference')):
-                referenced_table = f_type.split()[1].split('.')[0]
+                    links.append(dict(source=tablename, target = referenced_table))
 
-                links.append(dict(source=tablename, target = referenced_table))
-
-        nodes.append(dict(name=tablename, type="table", fields = fields))
+            nodes.append(dict(name=tablename, type="table", fields = fields))
 
     # d3 v4 allows individual modules to be specified.  The complete d3 library is included below.
-    response.files.append(URL('static','js/d3.min.js'))
-    response.files.append(URL('static','js/d3_graph.js'))
-    return dict(databases=databases, nodes=nodes, links=links)
+    response.files.append(URL('admin','static','js/d3.min.js'))
+    response.files.append(URL('admin','static','js/d3_graph.js'))
+    return dict(databases=databases, nodes=nodes, links=links)

applications/admin/views/appadmin.html

@@ -239,7 +239,7 @@
     {{=T("No databases in this application")}}
   {{else:}}    
     <div id="vis"></div>
-      <link rel="stylesheet" href="{{=URL('static','css/d3_graph.css')}}"/>
+      <link rel="stylesheet" href="{{=URL('admin','static','css/d3_graph.css')}}"/>
       <script>
         // Define the d3 input data
         {{from gluon.serializers import json }}

applications/admin/views/default/site.html

@@ -46,7 +46,7 @@
       </td>
       <td>
         <div class="btn-group">
-          <a class="btn dropdown-toggle" data-toggle="dropdown" href="#">
+          <a class="btn dropdown-toggle" data-toggle="dropdown">
         {{=T('Manage')}}
         <span class="caret"></span>
           </a>

applications/examples/controllers/appadmin.py

@@ -657,37 +657,36 @@ def d3_graph_model():
     Create a list of table dicts, called "nodes"
     """
     
-    data = {}
     nodes = []
     links = []
 
-    subgraphs = dict()
+    for database in databases:
+        db = eval_in_global_env(database)
+        for tablename in db.tables:
+            fields = []
+            for field in db[tablename]:
+                f_type = field.type
+                if not isinstance(f_type,str):
+                    disp = ' '
+                elif f_type == 'string':
+                    disp =  field.length
+                elif f_type == 'id':
+                    disp =  "PK"
+                elif f_type.startswith('reference') or \
+                    f_type.startswith('list:reference'):
+                    disp = "FK"
+                else:
+                    disp = ' '
+                fields.append(dict(name= field.name, type=field.type, disp = disp))
 
-    for tablename in db.tables:
-        fields = []
-        for field in db[tablename]:
-            f_type = field.type
-            if not isinstance(f_type,str):
-                disp = ' '
-            elif f_type == 'string':
-                disp =  field.length
-            elif f_type == 'id':
-                disp =  "PK"
-            elif f_type.startswith('reference') or \
-                f_type.startswith('list:reference'):
-                disp = "FK"
-            else:
-                disp = ' '
-            fields.append(dict(name= field.name, type=field.type, disp = disp))
+                if isinstance(f_type,str) and (
+                    f_type.startswith('reference') or
+                    f_type.startswith('list:reference')):
+                    referenced_table = f_type.split()[1].split('.')[0]
 
-            if isinstance(f_type,str) and (
-                f_type.startswith('reference') or
-                f_type.startswith('list:reference')):
-                referenced_table = f_type.split()[1].split('.')[0]
+                    links.append(dict(source=tablename, target = referenced_table))
 
-                links.append(dict(source=tablename, target = referenced_table))
-
-        nodes.append(dict(name=tablename, type="table", fields = fields))
+            nodes.append(dict(name=tablename, type="table", fields = fields))
 
     # d3 v4 allows individual modules to be specified.  The complete d3 library is included below.
     response.files.append(URL('admin','static','js/d3.min.js'))

applications/examples/views/appadmin.html

@@ -239,7 +239,7 @@
     {{=T("No databases in this application")}}
   {{else:}}    
     <div id="vis"></div>
-      <link rel="stylesheet" href="{{=URL('static','css/d3_graph.css')}}"/>
+      <link rel="stylesheet" href="{{=URL('admin','static','css/d3_graph.css')}}"/>
       <script>
         // Define the d3 input data
         {{from gluon.serializers import json }}

applications/welcome/controllers/appadmin.py

@@ -657,37 +657,36 @@ def d3_graph_model():
     Create a list of table dicts, called "nodes"
     """
     
-    data = {}
     nodes = []
     links = []
 
-    subgraphs = dict()
+    for database in databases:
+        db = eval_in_global_env(database)
+        for tablename in db.tables:
+            fields = []
+            for field in db[tablename]:
+                f_type = field.type
+                if not isinstance(f_type,str):
+                    disp = ' '
+                elif f_type == 'string':
+                    disp =  field.length
+                elif f_type == 'id':
+                    disp =  "PK"
+                elif f_type.startswith('reference') or \
+                    f_type.startswith('list:reference'):
+                    disp = "FK"
+                else:
+                    disp = ' '
+                fields.append(dict(name= field.name, type=field.type, disp = disp))
 
-    for tablename in db.tables:
-        fields = []
-        for field in db[tablename]:
-            f_type = field.type
-            if not isinstance(f_type,str):
-                disp = ' '
-            elif f_type == 'string':
-                disp =  field.length
-            elif f_type == 'id':
-                disp =  "PK"
-            elif f_type.startswith('reference') or \
-                f_type.startswith('list:reference'):
-                disp = "FK"
-            else:
-                disp = ' '
-            fields.append(dict(name= field.name, type=field.type, disp = disp))
+                if isinstance(f_type,str) and (
+                    f_type.startswith('reference') or
+                    f_type.startswith('list:reference')):
+                    referenced_table = f_type.split()[1].split('.')[0]
 
-            if isinstance(f_type,str) and (
-                f_type.startswith('reference') or
-                f_type.startswith('list:reference')):
-                referenced_table = f_type.split()[1].split('.')[0]
+                    links.append(dict(source=tablename, target = referenced_table))
 
-                links.append(dict(source=tablename, target = referenced_table))
-
-        nodes.append(dict(name=tablename, type="table", fields = fields))
+            nodes.append(dict(name=tablename, type="table", fields = fields))
 
     # d3 v4 allows individual modules to be specified.  The complete d3 library is included below.
     response.files.append(URL('admin','static','js/d3.min.js'))

applications/welcome/routes.example.py

@@ -20,8 +20,8 @@
 # YOU CAN COPY THIS FILE TO ANY APPLICATION'S ROOT DIRECTORY WITHOUT CHANGES!
 # ----------------------------------------------------------------------------------------------------------------------
 
-from fileutils import abspath
-from languages import read_possible_languages
+from gluon.fileutils import abspath
+from gluon.languages import read_possible_languages
 
 possible_languages = read_possible_languages(abspath('applications', app))
 # ----------------------------------------------------------------------------------------------------------------------

applications/welcome/static/js/web2py.js

@@ -12,6 +12,8 @@
         $.error('web2py.js has already been loaded!');
     }
 
+    var FORMDATA_IS_SUPPORTED = typeof(FormData) !== 'undefined';
+
     String.prototype.reverse = function () {
         return this.split('').reverse().join('');
     };
@@ -263,13 +265,17 @@
                 }
             });
             /* help preventing double form submission for normal form (not LOADed) */
-            $(doc).on('submit', 'form', function () {
-                var submit_button = $(this).find(web2py.formInputClickSelector);
-                web2py.disableElement(submit_button);
+            $(doc).on('submit', 'form', function (e) {
+                var submit_buttons = $(this).find(web2py.formInputClickSelector);
+                submit_buttons.each(function() {
+                    web2py.disableElement($(this));
+                })
                 /* safeguard in case the form doesn't trigger a refresh,
                 see https://github.com/web2py/web2py/issues/1100 */
                 setTimeout(function () {
-                    web2py.enableElement(submit_button);
+                    submit_buttons.each(function() {
+                        web2py.enableElement($(this));
+                    });
                 }, 5000);
             });
             doc.ajaxSuccess(function (e, xhr) {
@@ -320,7 +326,15 @@
                 form.submit(function (e) {
                     web2py.disableElement(form.find(web2py.formInputClickSelector));
                     web2py.hide_flash();
-                    web2py.ajax_page('post', url, form.serialize(), target, form);
+
+                    var formData;
+                    if (FORMDATA_IS_SUPPORTED) {
+                        formData = new FormData(form[0]); // Allows file uploads.
+                    } else {
+                        formData = form.serialize(); // Fallback for older browsers.
+                    }
+                    web2py.ajax_page('post', url, formData, target, form);
+
                     e.preventDefault();
                 });
                 form.on('click', web2py.formInputClickSelector, function (e) {
@@ -339,11 +353,18 @@
             if (web2py.isUndefined(element)) element = $(document);
             /* if target is not there, fill it with something that there isn't in the page*/
             if (web2py.isUndefined(target) || target === '') target = 'w2p_none';
+
+            /* processData and contentType must be set to false when passing a FormData
+               object to jQuery.ajax. */
+            var isFormData = Object.prototype.toString.call(data) === '[object FormData]';
+            var contentType = isFormData ? false : 'application/x-www-form-urlencoded; charset=UTF-8';
             if (web2py.fire(element, 'ajax:before', null, target)) { /*test a usecase, should stop here if returns false */
                 $.ajax({
                     'type': method,
                     'url': action,
                     'data': data,
+                    'processData': !isFormData,
+                    'contentType': contentType,
                     'beforeSend': function (xhr, settings) {
                         xhr.setRequestHeader('web2py-component-location', document.location);
                         xhr.setRequestHeader('web2py-component-element', target);
@@ -699,8 +720,9 @@
             });
         },
         /* Disables form elements:
+        - Does not disable elements with 'data-w2p_disable' attribute
         - Caches element value in 'w2p_enable_with' data store
-        - Replaces element text with value of 'data-disable-with' attribute
+        - Replaces element text with value of 'data-w2p_disable_with' attribute
         - Sets disabled property to true
         */
         disableFormElements: function (form) {
@@ -712,13 +734,15 @@
                 if (!web2py.isUndefined(disable)) {
                     return false;
                 }
-                if (web2py.isUndefined(disable_with)) {
-                    element.data('w2p_disable_with', element[method]());
+                if (!element.is(':file')) { // Altering file input values is not allowed.
+                    if (web2py.isUndefined(disable_with)) {
+                        element.data('w2p_disable_with', element[method]());
+                    }
+                    if (web2py.isUndefined(element.data('w2p_enable_with'))) {
+                        element.data('w2p_enable_with', element[method]());
+                    }
+                    element[method](element.data('w2p_disable_with'));
                 }
-                if (web2py.isUndefined(element.data('w2p_enable_with'))) {
-                    element.data('w2p_enable_with', element[method]());
-                }
-                element[method](element.data('w2p_disable_with'));
                 element.prop('disabled', true);
             });
         },

applications/welcome/views/appadmin.html

@@ -239,7 +239,7 @@
     {{=T("No databases in this application")}}
   {{else:}}    
     <div id="vis"></div>
-      <link rel="stylesheet" href="{{=URL('static','css/d3_graph.css')}}"/>
+      <link rel="stylesheet" href="{{=URL('admin','static','css/d3_graph.css')}}"/>
       <script>
         // Define the d3 input data
         {{from gluon.serializers import json }}

gluon/admin.py

@@ -54,7 +54,8 @@ def app_pack(app, request, raise_ex=False, filenames=None):
 
     """
     try:
-        if filenames is None: app_cleanup(app, request)
+        if filenames is None:
+            app_cleanup(app, request)
         filename = apath('../deposit/web2py.app.%s.w2p' % app, request)
         w2p_pack(filename, apath(app, request), filenames=filenames)
         return filename
@@ -104,7 +105,8 @@ def app_cleanup(app, request):
     if os.path.exists(path):
         for f in os.listdir(path):
             try:
-                if f[:1] != '.': os.unlink(os.path.join(path, f))
+                if f[:1] != '.':
+                    os.unlink(os.path.join(path, f))
             except IOError:
                 r = False
 
@@ -113,7 +115,8 @@ def app_cleanup(app, request):
     if os.path.exists(path):
         for f in os.listdir(path):
             try:
-                if f[:1] != '.': recursive_unlink(os.path.join(path, f))
+                if f[:1] != '.':
+                    recursive_unlink(os.path.join(path, f))
             except (OSError, IOError):
                 r = False
 
@@ -123,7 +126,8 @@ def app_cleanup(app, request):
         CacheOnDisk(folder=path).clear()
         for f in os.listdir(path):
             try:
-                if f[:1] != '.': recursive_unlink(os.path.join(path, f))
+                if f[:1] != '.':
+                    recursive_unlink(os.path.join(path, f))
             except (OSError, IOError):
                 r = False
     return r
@@ -175,10 +179,9 @@ def app_create(app, request, force=False, key=None, info=False):
             return False
     try:
         w2p_unpack('welcome.w2p', path)
-        for subfolder in [
-            'models', 'views', 'controllers', 'databases',
-            'modules', 'cron', 'errors', 'sessions', 'cache',
-            'languages', 'static', 'private', 'uploads']:
+        for subfolder in ['models', 'views', 'controllers', 'databases',
+                          'modules', 'cron', 'errors', 'sessions', 'cache',
+                          'languages', 'static', 'private', 'uploads']:
             subpath = os.path.join(path, subfolder)
             if not os.path.exists(subpath):
                 os.mkdir(subpath)
@@ -368,7 +371,7 @@ def unzip(filename, dir, subfolder=''):
     for name in sorted(zf.namelist()):
         if not name.startswith(subfolder):
             continue
-        #print name[n:]
+        # print name[n:]
         if name.endswith('/'):
             folder = os.path.join(dir, name[n:])
             if not os.path.exists(folder):
@@ -435,6 +438,7 @@ def add_path_first(path):
     if not global_settings.web2py_runtime_gae:
         site.addsitedir(path)
 
+
 def try_mkdir(path):
     if not os.path.exists(path):
         try:
@@ -444,11 +448,12 @@ def try_mkdir(path):
             else:
                 os.mkdir(path)
         except OSError as e:
-            if e.strerror == 'File exists': # In case of race condition.
+            if e.strerror == 'File exists':  # In case of race condition.
                 pass
             else:
                 raise e
 
+
 def create_missing_folders():
     if not global_settings.web2py_runtime_gae:
         for path in ('applications', 'deposit', 'site-packages', 'logs'):

gluon/authapi.py

@@ -20,10 +20,10 @@ DEFAULT = lambda: None
 class AuthAPI(object):
     """
     AuthAPI is a barebones Auth implementation which does not have a concept of
-    HTML forms or redirects, emailing or even an URL, you are responsible for 
+    HTML forms or redirects, emailing or even an URL, you are responsible for
     all that if you use it.
     The main Auth functions such as login, logout, register, profile are designed
-    in a Dict In -> Dict Out logic so, for instance, if you set 
+    in a Dict In -> Dict Out logic so, for instance, if you set
     registration_requires_verification you are responsible for sending the key to
     the user and even rolling back the transaction if you can't do it.
 
@@ -115,7 +115,7 @@ class AuthAPI(object):
             if auth.last_visit and auth.last_visit + delta > now:
                 self.user = auth.user
                 # this is a trick to speed up sessions to avoid many writes
-                if (now - auth.last_visit).seconds > (auth.expiration / 10):
+                if (now - auth.last_visit).seconds > (auth.expiration // 10):
                     auth.last_visit = now
             else:
                 self.user = None
@@ -245,13 +245,13 @@ class AuthAPI(object):
             migrate = db._migrate
         if fake_migrate is None:
             fake_migrate = db._fake_migrate
-        
+
         settings = self.settings
         if username is None:
             username = settings.use_username
         else:
             settings.use_username = username
-        
+
         if not self.signature:
             self.define_signature()
         if signature is True:
@@ -557,27 +557,43 @@ class AuthAPI(object):
         self.log_event(self.messages['del_membership_log'],
                        dict(user_id=user_id, group_id=group_id))
         ret = self.db(membership.user_id == user_id)(membership.group_id == group_id).delete()
-        if group_id in self.user_groups:
+        if group_id in self.user_groups and user_id == self.user_id:
             del self.user_groups[group_id]
         return ret
 
-    def has_membership(self, group_id=None, user_id=None, role=None):
+    def has_membership(self, group_id=None, user_id=None, role=None, cached=False):
         """
         Checks if user is member of group_id or role
+
+        NOTE: To avoid database query at each page load that use auth.has_membership, someone can use cached=True.
+              If cached is set to True has_membership() check group_id or role only against auth.user_groups variable
+              which is populated properly only at login time. This means that if an user membership change during a
+              given session the user has to log off and log in again in order to auth.user_groups to be properly
+              recreated and reflecting the user membership modification. There is one exception to this log off and
+              log in process which is in case that the user change his own membership, in this case auth.user_groups
+              can be properly update for the actual connected user because web2py has access to the proper session
+              user_groups variable. To make use of this exception someone has to place an "auth.update_groups()"
+              instruction in his app code to force auth.user_groups to be updated. As mention this will only work if it
+              the user itself that change it membership not if another user, let say an administrator, change someone
+              else's membership.
         """
-        group_id = group_id or self.id_group(role)
-        try:
-            group_id = int(group_id)
-        except:
-            group_id = self.id_group(group_id)  # interpret group_id as a role
         if not user_id and self.user:
             user_id = self.user.id
-        membership = self.table_membership()
-        if group_id and user_id and self.db((membership.user_id == user_id) &
-                                            (membership.group_id == group_id)).select():
-            r = True
+        if cached:
+            id_role = group_id or role
+            r = (user_id and id_role in self.user_groups.values()) or (user_id and id_role in self.user_groups)
         else:
-            r = False
+            group_id = group_id or self.id_group(role)
+            try:
+                group_id = int(group_id)
+            except:
+                group_id = self.id_group(group_id)  # interpret group_id as a role
+            membership = self.table_membership()
+            if group_id and user_id and self.db((membership.user_id == user_id) &
+                                                (membership.group_id == group_id)).select():
+                r = True
+            else:
+                r = False
         self.log_event(self.messages['has_membership_log'],
                        dict(user_id=user_id, group_id=group_id, check=r))
         return r
@@ -1012,7 +1028,7 @@ class AuthAPI(object):
                    ):
         """
         Verify a given registration_key actually exists in the user table.
-        Resets the key to empty string '' or 'pending' if 
+        Resets the key to empty string '' or 'pending' if
         setttings.registration_requires_approval is true.
 
         Keyword Args:

gluon/compileapp.py

@@ -41,11 +41,12 @@ import imp
 import logging
 import types
 from functools import reduce
-logger = logging.getLogger("web2py")
 from gluon import rewrite
 from gluon.custom_import import custom_import_install
 import py_compile
 
+logger = logging.getLogger("web2py")
+
 is_pypy = settings.global_settings.is_pypy
 is_gae = settings.global_settings.web2py_runtime_gae
 is_jython = settings.global_settings.is_jython
@@ -111,7 +112,7 @@ class mybuiltin(object):
     NOTE could simple use a dict and populate it,
     NOTE not sure if this changes things though if monkey patching import.....
     """
-    #__builtins__
+    # __builtins__
     def __getitem__(self, key):
         try:
             return getattr(builtin, key)
@@ -185,7 +186,7 @@ def LOAD(c=None, f='index', args=None, vars=None,
         else:
             statement = "$.web2py.component('%s','%s');" % (url, target)
         attr['_data-w2p_remote'] = url
-        if not target is None:
+        if target is not None:
             return DIV(content, **attr)
 
     else:
@@ -211,7 +212,8 @@ def LOAD(c=None, f='index', args=None, vars=None,
             request.env.path_info
         other_request.cid = target
         other_request.env.http_web2py_component_element = target
-        other_request.restful = types.MethodType(request.restful.__func__, other_request) # A bit nasty but needed to use LOAD on action decorates with @request.restful()
+        other_request.restful = types.MethodType(request.restful.__func__, other_request)
+        # A bit nasty but needed to use LOAD on action decorates with @request.restful()
         other_response.view = '%s/%s.%s' % (c, f, other_request.extension)
 
         other_environment = copy.copy(current.globalenv)  # NASTY
@@ -405,7 +407,7 @@ def build_environment(request, response, session, store_current=True):
     """
     Build the environment dictionary into which web2py files are executed.
     """
-    #h,v = html,validators
+    # h,v = html,validators
     environment = dict(_base_environment_)
 
     if not request.env:
@@ -418,7 +420,7 @@ def build_environment(request, response, session, store_current=True):
         r'^%s/%s/\w+\.py$' % (request.controller, request.function)
         ]
 
-    t = environment['T'] = translator(os.path.join(request.folder,'languages'),
+    t = environment['T'] = translator(os.path.join(request.folder, 'languages'),
                                       request.env.http_accept_language)
     c = environment['cache'] = Cache(request)
 
@@ -506,10 +508,12 @@ def compile_models(folder):
         save_pyc(filename)
         os.unlink(filename)
 
+
 def find_exposed_functions(data):
-    data = regex_longcomments.sub('',data)
+    data = regex_longcomments.sub('', data)
     return regex_expose.findall(data)
 
+
 def compile_controllers(folder):
     """
     Compiles all the controllers in the application specified by `folder`
@@ -524,16 +528,19 @@ def compile_controllers(folder):
             command = data + "\nresponse._vars=response._caller(%s)\n" % \
                 function
             filename = pjoin(folder, 'compiled',
-                             'controllers.%s.%s.py' % (fname[:-3],function))
+                             'controllers.%s.%s.py' % (fname[:-3], function))
             write_file(filename, command)
             save_pyc(filename)
             os.unlink(filename)
 
+
 def model_cmp(a, b, sep='.'):
     return cmp(a.count(sep), b.count(sep)) or cmp(a, b)
 
+
 def model_cmp_sep(a, b, sep=os.path.sep):
-    return model_cmp(a,b,sep)
+    return model_cmp(a, b, sep)
+
 
 def run_models_in(environment):
     """
@@ -544,7 +551,7 @@ def run_models_in(environment):
     request = current.request
     folder = request.folder
     c = request.controller
-    #f = environment['request'].function
+    # f = environment['request'].function
     response = current.response
 
     path = pjoin(folder, 'models')
@@ -557,9 +564,11 @@ def run_models_in(environment):
             models = sorted(listdir(path, '^\w+\.py$', 0, sort=False), model_cmp_sep)
     else:
         if compiled:
-            models = sorted(listdir(cpath, '^models[_.][\w.]+\.pyc$', 0), key=lambda f: '{0:03d}'.format(f.count('.')) + f)
+            models = sorted(listdir(cpath, '^models[_.][\w.]+\.pyc$', 0),
+                            key=lambda f: '{0:03d}'.format(f.count('.')) + f)
         else:
-            models = sorted(listdir(path, '^\w+\.py$', 0, sort=False), key=lambda f: '{0:03d}'.format(f.count(os.path.sep)) + f)
+            models = sorted(listdir(path, '^\w+\.py$', 0, sort=False),
+                            key=lambda f: '{0:03d}'.format(f.count(os.path.sep)) + f)
 
     models_to_run = None
     for model in models:
@@ -570,10 +579,10 @@ def run_models_in(environment):
         if models_to_run:
             if compiled:
                 n = len(cpath)+8
-                fname = model[n:-4].replace('.','/')+'.py'
+                fname = model[n:-4].replace('.', '/')+'.py'
             else:
                 n = len(path)+1
-                fname = model[n:].replace(os.path.sep,'/')
+                fname = model[n:].replace(os.path.sep, '/')
             if not regex.search(fname) and c != 'appadmin':
                 continue
             elif compiled:
@@ -583,6 +592,7 @@ def run_models_in(environment):
             ccode = getcfs(model, model, f)
             restricted(ccode, environment, layer=model)
 
+
 def run_controller_in(controller, function, environment):
     """
     Runs the controller.function() (for the app specified by
@@ -596,13 +606,13 @@ def run_controller_in(controller, function, environment):
     badc = 'invalid controller (%s/%s)' % (controller, function)
     badf = 'invalid function (%s/%s)' % (controller, function)
     if os.path.exists(cpath):
-        filename = pjoin(cpath, 'controllers.%s.%s.pyc'
-                         % (controller, function))
-        if not os.path.exists(filename):
+        filename = pjoin(cpath, 'controllers.%s.%s.pyc' % (controller, function))
+        try:
+            ccode = getcfs(filename, filename, lambda: read_pyc(filename))
+        except IOError:
             raise HTTP(404,
                        rewrite.THREAD_LOCAL.routes.error_message % badf,
                        web2py_error=badf)
-        ccode = getcfs(filename, filename, lambda: read_pyc(filename))
     elif function == '_TEST':
         # TESTING: adjust the path to include site packages
         from gluon.settings import global_settings
@@ -623,15 +633,15 @@ def run_controller_in(controller, function, environment):
         code += TEST_CODE
         ccode = compile2(code, filename)
     else:
-        filename = pjoin(folder, 'controllers/%s.py'
-                                 % controller)
-        if not os.path.exists(filename):
+        filename = pjoin(folder, 'controllers/%s.py' % controller)
+        try:
+            code = getcfs(filename, filename, lambda: read_file(filename))
+        except IOError:
             raise HTTP(404,
                        rewrite.THREAD_LOCAL.routes.error_message % badc,
                        web2py_error=badc)
-        code = getcfs(filename, filename, lambda: read_file(filename))
         exposed = find_exposed_functions(code)
-        if not function in exposed:
+        if function not in exposed:
             raise HTTP(404,
                        rewrite.THREAD_LOCAL.routes.error_message % badf,
                        web2py_error=badf)
@@ -666,6 +676,7 @@ def run_view_in(environment):
     badv = 'invalid view (%s)' % view
     patterns = response.get('generic_patterns')
     layer = None
+    scode = None
     if patterns:
         regex = re_compile('|'.join(map(fnmatch.translate, patterns)))
         short_action = '%(controller)s/%(function)s.%(extension)s' % request
@@ -678,7 +689,7 @@ def run_view_in(environment):
         layer = 'file stream'
     else:
         filename = pjoin(folder, 'views', view)
-        if os.path.exists(cpath): # compiled views
+        if os.path.exists(cpath):  # compiled views
             x = view.replace('/', '.')
             files = ['views.%s.pyc' % x]
             is_compiled = os.path.exists(pjoin(cpath, files[0]))
@@ -705,16 +716,21 @@ def run_view_in(environment):
             raise HTTP(404,
                        rewrite.THREAD_LOCAL.routes.error_message % badv,
                        web2py_error=badv)
-        layer = filename
-        # Compile the template
-        ccode = parse_template(view,
-                               pjoin(folder, 'views'),
-                               context=environment)
 
-    restricted(ccode, environment, layer=layer)
+        # if the view is not compiled
+        if not layer:
+            # Parse template
+            scode = parse_template(view,
+                                   pjoin(folder, 'views'),
+                                   context=environment)
+            # Compile template
+            ccode = compile2(scode, filename)
+        layer = filename
+    restricted(ccode, environment, layer=layer, scode=scode)
     # parse_template saves everything in response body
     return environment['response'].body.getvalue()
 
+
 def remove_compiled_application(folder):
     """
     Deletes the folder `compiled` containing the compiled application.

gluon/contenttype.py

@@ -330,7 +330,7 @@ CONTENT_TYPE = {
     '.lha': 'application/x-lha',
     '.lhs': 'text/x-literate-haskell',
     '.lhz': 'application/x-lhz',
-    '.load' : 'text/html',
+    '.load': 'text/html',
     '.log': 'text/x-log',
     '.lrz': 'application/x-lrzip',
     '.ltx': 'text/x-tex',
@@ -823,7 +823,7 @@ CONTENT_TYPE = {
     '.xsd': 'application/xml',
     '.xsl': 'application/xslt+xml',
     '.xslfo': 'text/x-xslfo',
-    '.xslm' : 'application/vnd.ms-excel.sheet.macroEnabled.12',
+    '.xslm': 'application/vnd.ms-excel.sheet.macroEnabled.12',
     '.xslt': 'application/xslt+xml',
     '.xspf': 'application/xspf+xml',
     '.xul': 'application/vnd.mozilla.xul+xml',
@@ -843,7 +843,7 @@ def contenttype(filename, default='text/plain'):
     """
     Returns the Content-Type string matching extension of the given filename.
     """
-    filename=to_native(filename)
+    filename = to_native(filename)
     i = filename.rfind('.')
     if i >= 0:
         default = CONTENT_TYPE.get(filename[i:].lower(), default)

gluon/contrib/heroku.py

@@ -7,15 +7,13 @@ db = get_db()
 """
 import os
 from gluon import *
-from pydal.adapters import ADAPTERS, PostgreSQLAdapter
-from pydal.helpers.classes import UseDatabaseStoredFile
+from pydal.adapters import adapters, PostgrePsyco
+from pydal.helpers.classes import DatabaseStoredFile
 
-class HerokuPostgresAdapter(UseDatabaseStoredFile,PostgreSQLAdapter):
-    drivers = ('psycopg2',)
+@adapters.register_for('postgres')
+class HerokuPostgresAdapter(DatabaseStoredFile, PostgrePsyco):
     uploads_in_blob = True
 
-ADAPTERS['postgres'] = HerokuPostgresAdapter
-
 def get_db(name = None, pool_size=10):
     if not name:
         names = [n for n in os.environ.keys()

gluon/contrib/imageutils.py

@@ -45,7 +45,7 @@ class RESIZE(object):
                 background = Image.new('RGBA', (self.nx, self.ny), (255, 255, 255, 0))
                 background.paste(
                     img,
-                    ((self.nx - img.size[0]) / 2, (self.ny - img.size[1]) / 2))
+                    ((self.nx - img.size[0]) // 2, (self.ny - img.size[1]) // 2))
                 background.save(s, 'JPEG', quality=self.quality)
             else:
                 img.save(s, 'JPEG', quality=self.quality)

gluon/contrib/login_methods/cas_auth.py

@@ -49,7 +49,8 @@ class CasAuth(object):
                            email=lambda v: v.get('email', None),
                            user_id=lambda v: v['user']),
                  casversion=1,
-                 casusername='cas:user'
+                 casusername='cas:user',
+		 change_password_url=None
                  ):
         self.urlbase = urlbase
         self.cas_login_url = "%s/%s" % (self.urlbase, actions[0])
@@ -64,6 +65,9 @@ class CasAuth(object):
                               #vars=current.request.vars,
                               scheme=True)
 
+	# URL to let users change their password in the IDP system
+        self.cas_change_password_url = change_password_url
+
     def login_url(self, next="/"):
         current.session.token = self._CAS_login()
         return next
@@ -74,6 +78,10 @@ class CasAuth(object):
         self._CAS_logout()
         return next
 
+    def change_password_url(self, next="/"):
+        self._CAS_change_password()
+        return next
+
     def get_user(self):
         user = current.session.token
         if user:
@@ -135,3 +143,6 @@ class CasAuth(object):
         redirects to the CAS logout page
         """
         redirect("%s?service=%s" % (self.cas_logout_url, self.cas_my_url))
+
+    def _CAS_change_password(self):
+        redirect(self.cas_change_password_url)

gluon/contrib/login_methods/saml2_auth.py

@@ -145,10 +145,16 @@ class Saml2Auth(object):
             username=lambda v:v['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/upn'][0],
             email=lambda v:v['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/upn'][0],
             user_id=lambda v:v['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/upn'][0],
-            )):
+            ), logout_url=None, change_password_url=None):
         self.config_file = config_file
         self.maps = maps
 
+	# URL for redirecting users to when they sign out
+        self.saml_logout_url = logout_url
+
+        # URL to let users change their password in the IDP system
+        self.saml_change_password_url = change_password_url
+
     def login_url(self, next="/"):
         d = saml2_handler(current.session, current.request)
         if 'url' in d:
@@ -170,6 +176,12 @@ class Saml2Auth(object):
 
     def logout_url(self, next="/"):
         current.session.saml2_info = None
+        current.session.auth = None
+        self._SAML_logout()
+        return next
+
+    def change_password_url(self, next="/"):
+        self._SAML_change_password()
         return next
 
     def get_user(self):        
@@ -180,3 +192,13 @@ class Saml2Auth(object):
                 d[key] = self.maps[key](user)
             return d
         return None
+
+    def _SAML_logout(self):
+        """
+        exposed SAML.logout()
+        redirects to the SAML logout page
+        """
+        redirect(self.saml_logout_url)
+
+    def _SAML_change_password(self):
+        redirect(self.saml_change_password_url)

gluon/contrib/markmin/markmin2html.py

@@ -5,12 +5,19 @@
 # license MIT/BSD/GPL
 from __future__ import print_function
 import re
+import sys
 import urllib
-from gluon._compat import maketrans, urllib_quote, unicodeT, to_bytes, to_native, xrange
-from gluon.utils import local_html_escape as escape
-from ast import parse as ast_parse
 import ast
 
+PY2 = sys.version_info[0] == 2
+
+if PY2:
+    from urllib import quote as urllib_quote
+    from string import maketrans
+else:
+    from urllib.parse import quote as urllib_quote
+    maketrans = str.maketrans
+
 
 """
 TODO: next version should use MathJax
@@ -564,6 +571,29 @@ ttab_in = maketrans("'`:*~\\[]{}@$+-.#\n", '\x0b\x0c\x0e\x0f\x10\x11\x12\x13\x14
 ttab_out = maketrans('\x0b\x0c\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x05', "'`:*~\\[]{}@$+-.#\n")
 regex_quote = re.compile('(?P<name>\w+?)\s*\=\s*')
 
+def local_html_escape(data, quote=False):
+    """
+    Works with bytes.
+    Replace special characters "&", "<" and ">" to HTML-safe sequences.
+    If the optional flag quote is true (the default), the quotation mark
+    characters, both double quote (") and single quote (') characters are also
+    translated.
+    """
+    if PY2:
+        import cgi
+        data = cgi.escape(data, quote)
+        return data.replace("'", "&#x27;") if quote else data
+    else:
+        import html
+        if isinstance(data, str):
+            return html.escape(data, quote=quote)
+        data = data.replace(b"&", b"&amp;")  # Must be done first!                                                                                           
+        data = data.replace(b"<", b"&lt;")
+        data = data.replace(b">", b"&gt;")
+        if quote:
+            data = data.replace(b'"', b"&quot;")
+            data = data.replace(b'\'', b"&#x27;")
+        return data
 
 def make_dict(b):
     return '{%s}' % regex_quote.sub("'\g<name>':", b)
@@ -579,7 +609,7 @@ def safe_eval(node_or_string, env):
     _safe_names = {'None': None, 'True': True, 'False': False}
     _safe_names.update(env)
     if isinstance(node_or_string, basestring):
-        node_or_string = ast_parse(node_or_string, mode='eval')
+        node_or_string = ast.parse(node_or_string, mode='eval')
     if isinstance(node_or_string, ast.Expression):
         node_or_string = node_or_string.body
 
@@ -599,14 +629,14 @@ def safe_eval(node_or_string, env):
             if node.id in _safe_names:
                 return _safe_names[node.id]
         elif isinstance(node, ast.BinOp) and \
-                isinstance(node.op, (Add, Sub)) and \
-                isinstance(node.right, Num) and \
+                isinstance(node.op, (ast.Add, ast.Sub)) and \
+                isinstance(node.right, ast.Num) and \
                 isinstance(node.right.n, complex) and \
-                isinstance(node.left, Num) and \
+                isinstance(node.left, ast.Num) and \
                 isinstance(node.left.n, (int, long, float)):
             left = node.left.n
             right = node.right.n
-            if isinstance(node.op, Add):
+            if isinstance(node.op, ast.Add):
                 return left + right
             else:
                 return left - right
@@ -765,7 +795,7 @@ def render(text,
     '<table><tbody><tr class="first"><td>a</td><td>b</td></tr><tr class="even"><td>c</td><td>d</td></tr></tbody></table>'
 
     >>> render("----\\nhello world\\n----\\n")
-    '<blockquote>hello world</blockquote>'
+    '<blockquote><p>hello world</p></blockquote>'
 
     >>> render('[[myanchor]]')
     '<p><span class="anchor" id="markmin_myanchor"></span></p>'
@@ -946,7 +976,8 @@ def render(text,
     if protolinks == "default":
         protolinks = protolinks_simple
     pp = '\n' if pretty_print else ''
-    text = to_native(text)
+    text = text if text is None or isinstance(text, str) else text.decode('utf8', 'strict')
+
     if not (isinstance(text, str)):
         text = str(text or '')
     text = regex_backslash.sub(lambda m: m.group(1).translate(ttab_in), text)
@@ -994,7 +1025,7 @@ def render(text,
         return LINK
 
     text = regex_link.sub(mark_link, text)
-    text = escape(text)
+    text = local_html_escape(text)
 
     if protolinks:
         text = regex_proto.sub(lambda m: protolinks(*m.group('p', 'k')), text)
@@ -1035,7 +1066,7 @@ def render(text,
             if pend and mtag == '.':  # paragraph in a list:
                 out.append(etags.pop())
                 ltags.pop()
-            for i in xrange(lent - lev):
+            for i in range(lent - lev):
                 out.append('<' + tag + '>' + pp)
                 etags.append('</' + tag + '>' + pp)
                 lev += 1
@@ -1044,7 +1075,7 @@ def render(text,
         elif lent == lev:
             if tlev[-1] != tag:
                 # type of list is changed (ul<=>ol):
-                for i in xrange(ltags.count(lent)):
+                for i in range(ltags.count(lent)):
                     ltags.pop()
                     out.append(etags.pop())
                 tlev[-1] = tag
@@ -1209,7 +1240,7 @@ def render(text,
                 s = '<blockquote%s%s>%s</blockquote>%s' \
                     % (t_cls,
                        t_id,
-                       '\n'.join(strings[bq_begin:lineno]), pp)
+                       render('\n'.join(strings[bq_begin:lineno])), pp)
                 mtag = 'q'
         else:
             s = '<hr />'
@@ -1322,10 +1353,10 @@ def render(text,
         t, a, k, p, w = m.group('t', 'a', 'k', 'p', 'w')
         if not k:
             return m.group(0)
-        k = escape(k)
+        k = local_html_escape(k)
         t = t or ''
         style = 'width:%s' % w if w else ''
-        title = ' title="%s"' % escape(a).replace(META, DISABLED_META) if a else ''
+        title = ' title="%s"' % local_html_escape(a).replace(META, DISABLED_META) if a else ''
         p_begin = p_end = ''
         if p == 'center':
             p_begin = '<p style="text-align:center">'
@@ -1349,7 +1380,7 @@ def render(text,
                        autolinks, protolinks, class_prefix, id_prefix, pretty_print)
             return '<%(p)s controls="controls"%(title)s%(style)s><source src="%(k)s" />%(t)s</%(p)s>' \
                    % dict(p=p, title=title, style=style, k=k, t=t)
-        alt = ' alt="%s"' % escape(t).replace(META, DISABLED_META) if t else ''
+        alt = ' alt="%s"' % local_html_escape(t).replace(META, DISABLED_META) if t else ''
         return '%(begin)s<img src="%(k)s"%(alt)s%(title)s%(style)s />%(end)s' \
                % dict(begin=p_begin, k=k, alt=alt, title=title, style=style, end=p_end)
 
@@ -1358,12 +1389,12 @@ def render(text,
         if not k and not t:
             return m.group(0)
         t = t or ''
-        a = escape(a) if a else ''
+        a = local_html_escape(a) if a else ''
         if k:
             if '#' in k and ':' not in k.split('#')[0]:
                 # wikipage, not external url
                 k = k.replace('#', '#' + id_prefix)
-            k = escape(k)
+            k = local_html_escape(k)
             title = ' title="%s"' % a.replace(META, DISABLED_META) if a else ''
             target = ' target="_blank"' if p == 'popup' else ''
             t = render(t, {}, {}, 'br', URL, environment, latex, None,
@@ -1373,7 +1404,7 @@ def render(text,
         if t == 'NEWLINE' and not a:
             return '<br />' + pp
         return '<span class="anchor" id="%s">%s</span>' % (
-            escape(id_prefix + t),
+            local_html_escape(id_prefix + t),
             render(a, {}, {}, 'br', URL,
                    environment, latex, autolinks,
                    protolinks, class_prefix,
@@ -1399,7 +1430,7 @@ def render(text,
     def expand_meta(m):
         code, b, p, s = segments.pop(0)
         if code is None or m.group() == DISABLED_META:
-            return escape(s)
+            return local_html_escape(s)
         if b in extra:
             if code[:1] == '\n':
                 code = code[1:]
@@ -1411,7 +1442,7 @@ def render(text,
                 return str(extra[b](code))
         elif b == 'cite':
             return '[' + ','.join('<a href="#%s" class="%s">%s</a>' %
-                                  (id_prefix + d, b, d) for d in escape(code).split(',')) + ']'
+                                  (id_prefix + d, b, d) for d in local_html_escape(code).split(',')) + ']'
         elif b == 'latex':
             return LATEX % urllib_quote(code)
         elif b in html_colors:
@@ -1426,12 +1457,12 @@ def render(text,
                    % (fg, bg, render(code, {}, {}, 'br', URL, environment, latex,
                                      autolinks, protolinks, class_prefix, id_prefix, pretty_print))
         cls = ' class="%s%s"' % (class_prefix, b) if b and b != 'id' else ''
-        id = ' id="%s%s"' % (id_prefix, escape(p)) if p else ''
+        id = ' id="%s%s"' % (id_prefix, local_html_escape(p)) if p else ''
         beg = (code[:1] == '\n')
         end = [None, -1][code[-1:] == '\n']
         if beg and end:
-            return '<pre><code%s%s>%s</code></pre>%s' % (cls, id, escape(code[1:-1]), pp)
-        return '<code%s%s>%s</code>' % (cls, id, escape(code[beg:end]))
+            return '<pre><code%s%s>%s</code></pre>%s' % (cls, id, local_html_escape(code[1:-1]), pp)
+        return '<code%s%s>%s</code>' % (cls, id, local_html_escape(code[beg:end]))
 
     text = regex_expand_meta.sub(expand_meta, text)
 

gluon/contrib/minify/cssmin.py

@@ -10,8 +10,11 @@ Original author: Zachary Voase
 Modified for inclusion into web2py by: Ross Peoples <ross.peoples@gmail.com>
 """
 
+try:
+    from StringIO import StringIO
+except ImportError:
+    from io import StringIO
 
-from StringIO import StringIO  # The pure-Python StringIO supports unicode.
 import re
 
 

gluon/contrib/minify/minify.py

@@ -8,30 +8,40 @@ Created by: Ross Peoples <ross.peoples@gmail.com>
 Modified by: Massimo Di Pierro <massimo.dipierro@gmail.com>
 """
 
-import cssmin
-import jsmin
+from . import cssmin
+from . import jsmin
 import os
 import hashlib
 import re
+import sys
+PY2 = sys.version_info[0] == 2
 
+if PY2:
+    hashlib_md5 = hashlib.md5
+else:
+    hashlib_md5 = lambda s: hashlib.md5(bytes(s, 'utf8'))
+
+def open_py23(filename, mode):
+    if PY2:
+        f = open(filename, mode + 'b')
+    else:
+        f = open(filename, mode, encoding="utf8")
+    return f
 
 def read_binary_file(filename):
-    f = open(filename, 'rb')
+    f = open_py23(filename, 'r')
     data = f.read()
     f.close()
     return data
 
-
 def write_binary_file(filename, data):
-    f = open(filename, 'wb')
+    f = open_py23(filename, 'w')
     f.write(data)
     f.close()
 
-
 def fix_links(css, static_path):
     return re.sub(r'url\((["\'])\.\./', 'url(\\1' + static_path, css)
 
-
 def minify(files, path_info, folder, optimize_css, optimize_js,
            ignore_concat=[],
            ignore_minify=['/jquery.js', '/anytime.js']):
@@ -109,7 +119,7 @@ def minify(files, path_info, folder, optimize_css, optimize_js,
                     js.append(contents)
             else:
                 js.append(filename)
-    dest_key = hashlib.md5(repr(processed)).hexdigest()
+    dest_key = hashlib_md5(repr(processed)).hexdigest()
     if css and concat_css:
         css = '\n\n'.join(contents for contents in css)
         if not inline_css:

gluon/contrib/user_agent_parser.py

@@ -673,3 +673,23 @@ def simple_detect(agent):
     if os_version:
         os = " ".join((os, os_version))
     return os, browser
+
+
+class mobilize(object):  
+    """
+    Decorator for controller functions so they use different views for mobile devices.
+
+    WARNING: If you update httpagentparser make sure to leave mobilize for
+    backwards compatibility.
+    """
+    def __init__(self, func):
+        self.func = func
+    
+    def __call__(self):
+        from gluon import current
+        user_agent = current.request.user_agent()
+        if user_agent.is_mobile:
+            items = current.response.view.split('.')
+            items.insert(-1, 'mobile')
+            current.response.view = '.'.join(items)
+        return self.func()

gluon/contrib/websocket_messaging.py

@@ -94,32 +94,10 @@ import optparse
 import time
 import sys
 import gluon.utils
-
-if (sys.version_info[0] == 2):
-    from urllib import urlencode, urlopen
-    def to_bytes(obj, charset='utf-8', errors='strict'):
-        if obj is None:
-            return None
-        if isinstance(obj, (bytes, bytearray, buffer)):
-            return bytes(obj)
-        if isinstance(obj, unicode):
-            return obj.encode(charset, errors)
-        raise TypeError('Expected bytes')
-else:
-    from urllib.request import urlopen
-    from urllib.parse import urlencode
-    def to_bytes(obj, charset='utf-8', errors='strict'):
-        if obj is None:
-            return None
-        if isinstance(obj, (bytes, bytearray, memoryview)):
-            return bytes(obj)
-        if isinstance(obj, str):
-            return obj.encode(charset, errors)
-        raise TypeError('Expected bytes')
+from gluon._compat import to_native, to_bytes, urlencode, urlopen
 
 listeners, names, tokens = {}, {}, {}
 
-
 def websocket_send(url, message, hmac_key=None, group='default'):
     sig = hmac_key and hmac.new(to_bytes(hmac_key), to_bytes(message)).hexdigest() or ''
     params = urlencode(
@@ -138,8 +116,8 @@ class PostHandler(tornado.web.RequestHandler):
         if hmac_key and not 'signature' in self.request.arguments:
             self.send_error(401)
         if 'message' in self.request.arguments:
-            message = self.request.arguments['message'][0]
-            group = self.request.arguments.get('group', ['default'])[0]
+            message = self.request.arguments['message'][0].decode(encoding='UTF-8')
+            group = self.request.arguments.get('group', ['default'])[0].decode(encoding='UTF-8')
             print('%s:MESSAGE to %s:%s' % (time.time(), group, message))
             if hmac_key:
                 signature = self.request.arguments['signature'][0]

gluon/dal.py

@@ -14,6 +14,12 @@ from pydal import DAL as DAL
 from pydal import Field
 from pydal.objects import Row, Rows, Table, Query, Set, Expression
 from pydal import SQLCustomType, geoPoint, geoLine, geoPolygon
+from pydal.migrator import Migrator, InDBMigrator
+from gluon.serializers import custom_json, xml
+from gluon.utils import web2py_uuid
+from gluon import sqlhtml
+from pydal.drivers import DRIVERS
+
 
 def _default_validators(db, field):
     """
@@ -78,14 +84,10 @@ def _default_validators(db, field):
     if (field.notnull or field.unique) and field_type not in excluded_fields:
         requires.insert(0, validators.IS_NOT_EMPTY())
     elif not field.notnull and not field.unique and requires:
-        requires[0] = validators.IS_EMPTY_OR(requires[0], null='' if field.type in ('string', 'text', 'password') else None)
+        requires[0] = \
+            validators.IS_EMPTY_OR(requires[0], null='' if field.type in ('string', 'text', 'password') else None)
     return requires
 
-from gluon.serializers import custom_json, xml
-from gluon.utils import web2py_uuid
-from gluon import sqlhtml
-
-
 DAL.serializers = {'json': custom_json, 'xml': xml}
 DAL.validators_method = _default_validators
 DAL.uuid = lambda x: web2py_uuid()
@@ -96,8 +98,7 @@ DAL.representers = {
 DAL.Field = Field
 DAL.Table = Table
 
-#: add web2py contrib drivers to pyDAL
-from pydal.drivers import DRIVERS
+# add web2py contrib drivers to pyDAL
 if not DRIVERS.get('pymysql'):
     try:
         from .contrib import pymysql

gluon/decoder.py

@@ -15,13 +15,13 @@ import codecs
 # None represents a potentially variable byte. "##" in the XML spec...
 autodetect_dict = {  # bytepattern     : ("name",
                                          (0x00, 0x00, 0xFE, 0xFF): ("ucs4_be"),
-    (0xFF, 0xFE, 0x00, 0x00): ("ucs4_le"),
-    (0xFE, 0xFF, None, None): ("utf_16_be"),
-    (0xFF, 0xFE, None, None): ("utf_16_le"),
-    (0x00, 0x3C, 0x00, 0x3F): ("utf_16_be"),
-    (0x3C, 0x00, 0x3F, 0x00): ("utf_16_le"),
-    (0x3C, 0x3F, 0x78, 0x6D): ("utf_8"),
-    (0x4C, 0x6F, 0xA7, 0x94): ("EBCDIC")
+                                         (0xFF, 0xFE, 0x00, 0x00): ("ucs4_le"),
+                                         (0xFE, 0xFF, None, None): ("utf_16_be"),
+                                         (0xFF, 0xFE, None, None): ("utf_16_le"),
+                                         (0x00, 0x3C, 0x00, 0x3F): ("utf_16_be"),
+                                         (0x3C, 0x00, 0x3F, 0x00): ("utf_16_le"),
+                                         (0x3C, 0x3F, 0x78, 0x6D): ("utf_8"),
+                                         (0x4C, 0x6F, 0xA7, 0x94): ("EBCDIC")
 }
 
 
@@ -36,10 +36,10 @@ def autoDetectXMLEncoding(buffer):
     # buffer at once but otherwise we'd have to decode a character at
     # a time looking for the quote character...that's a pain
 
-    encoding = "utf_8"    # according to the XML spec, this is the default
-                          # this code successively tries to refine the default
-                          # whenever it fails to refine, it falls back to
-                          # the last place encoding was set.
+    encoding = "utf_8"
+    # according to the XML spec, this is the default this code successively tries to refine the default
+    # whenever it fails to refine, it falls back to the last place encoding was set.
+
     if len(buffer) >= 4:
         bytes = (byte1, byte2, byte3, byte4) = tuple(map(ord, buffer[0:4]))
         enc_info = autodetect_dict.get(bytes, None)
@@ -51,8 +51,7 @@ def autoDetectXMLEncoding(buffer):
         enc_info = None
 
     if enc_info:
-        encoding = enc_info  # we've got a guess... these are
-                             #the new defaults
+        encoding = enc_info  # we've got a guess... these are the new defaults
 
         # try to find a more precise encoding using xml declaration
         secret_decoder_ring = codecs.lookup(encoding)[1]

gluon/fileutils.py

@@ -415,10 +415,10 @@ def fix_newlines(path):
 |\r|
 )''')
     for filename in listdir(path, '.*\.(py|html)$', drop=False):
-        rdata = read_file(filename, 'rb')
+        rdata = read_file(filename, 'r')
         wdata = regex.sub('\n', rdata)
         if wdata != rdata:
-            write_file(filename, wdata, 'wb')
+            write_file(filename, wdata, 'w')
 
 
 def copystream(

gluon/globals.py

@@ -13,7 +13,8 @@ Contains the classes for the global used variables:
 - Session
 
 """
-from gluon._compat import pickle, StringIO, copyreg, Cookie, urlparse, PY2, iteritems, to_unicode, to_native, unicodeT, long
+from gluon._compat import pickle, StringIO, copyreg, Cookie, urlparse, PY2, iteritems, to_unicode, to_native, \
+    unicodeT, long, hashlib_md5, urllib_quote
 from gluon.storage import Storage, List
 from gluon.streamer import streamer, stream_file_or_304_or_206, DEFAULT_CHUNK_SIZE
 from gluon.contenttype import contenttype
@@ -30,7 +31,7 @@ from gluon.fileutils import copystream
 import hashlib
 from pydal.contrib import portalocker
 from pickle import Pickler, MARK, DICT, EMPTY_DICT
-#from types import DictionaryType
+# from types import DictionaryType
 import datetime
 import re
 import os
@@ -48,7 +49,7 @@ PAST = 'Sat, 1-Jan-1971 00:00:00'
 FUTURE = 'Tue, 1-Dec-2999 23:59:59'
 
 try:
-    #FIXME PY3
+    # FIXME PY3
     from gluon.contrib.minify import minify
     have_minify = True
 except ImportError:
@@ -79,6 +80,7 @@ template_mapping = {
     'js:inline': js_inline
 }
 
+
 # IMPORTANT:
 # this is required so that pickled dict(s) and class.__dict__
 # are sorted and web2py can detect without ambiguity when a session changes
@@ -95,6 +97,7 @@ else:
     SortingPickler.dispatch_table = copyreg.dispatch_table.copy()
     SortingPickler.dispatch_table[dict] = SortingPickler.save_dict
 
+
 def sorting_dumps(obj, protocol=None):
     file = StringIO()
     SortingPickler(file, protocol).dump(obj)
@@ -120,7 +123,7 @@ def copystream_progress(request, chunk_size=10 ** 5):
         dest = tempfile.NamedTemporaryFile()
     except NotImplementedError:  # and GAE this
         dest = tempfile.TemporaryFile()
-    if not 'X-Progress-ID' in request.get_vars:
+    if 'X-Progress-ID' not in request.get_vars:
         copystream(source, dest, size, chunk_size)
         return dest
     cache_key = 'X-Progress-ID:' + request.get_vars['X-Progress-ID']
@@ -198,7 +201,8 @@ class Request(Storage):
         """Takes the QUERY_STRING and unpacks it to get_vars
         """
         query_string = self.env.get('query_string', '')
-        dget = urlparse.parse_qs(query_string, keep_blank_values=1)  # Ref: https://docs.python.org/2/library/cgi.html#cgi.parse_qs
+        dget = urlparse.parse_qs(query_string, keep_blank_values=1)
+        # Ref: https://docs.python.org/2/library/cgi.html#cgi.parse_qs
         get_vars = self._get_vars = Storage(dget)
         for (key, value) in iteritems(get_vars):
             if isinstance(value, list) and len(value) == 1:
@@ -228,8 +232,7 @@ class Request(Storage):
             body.seek(0)
 
         # parse POST variables on POST, PUT, BOTH only in post_vars
-        if (body and not is_json
-            and env.request_method in ('POST', 'PUT', 'DELETE', 'BOTH')):
+        if body and not is_json and env.request_method in ('POST', 'PUT', 'DELETE', 'BOTH'):
             query_string = env.pop('QUERY_STRING', None)
             dpost = cgi.FieldStorage(fp=body, environ=env, keep_blank_values=1)
             try:
@@ -328,11 +331,16 @@ class Request(Storage):
         user_agent = session._user_agent
         if user_agent:
             return user_agent
-        user_agent = user_agent_parser.detect(self.env.http_user_agent)
+        http_user_agent = self.env.http_user_agent
+        user_agent = user_agent_parser.detect(http_user_agent)
         for key, value in user_agent.items():
             if isinstance(value, dict):
                 user_agent[key] = Storage(value)
-        user_agent = session._user_agent = Storage(user_agent)
+        user_agent = Storage(user_agent)
+        user_agent.is_mobile = 'Mobile' in http_user_agent
+        user_agent.is_tablet = 'Tablet' in http_user_agent
+        session._user_agent = user_agent 
+        
         return user_agent
 
     def requires_https(self):
@@ -355,7 +363,7 @@ class Request(Storage):
             def f(_action=action, *a, **b):
                 request.is_restful = True
                 env = request.env
-                is_json = env.content_type=='application/json'
+                is_json = env.content_type == 'application/json'
                 method = env.request_method
                 if not ignore_extension and len(request.args) and '.' in request.args[-1]:
                     request.args[-1], _, request.extension = request.args[-1].rpartition('.')
@@ -451,13 +459,13 @@ class Response(Storage):
         for meta in iteritems((self.meta or {})):
             k, v = meta
             if isinstance(v, dict):
-                s += '<meta' + ''.join(' %s="%s"' % (xmlescape(key), to_native(xmlescape(v[key]))) for key in v) +' />\n'
+                s += '<meta' + ''.join(' %s="%s"' % (xmlescape(key),
+                                                     to_native(xmlescape(v[key]))) for key in v) + ' />\n'
             else:
                 s += '<meta name="%s" content="%s" />\n' % (k, to_native(xmlescape(v)))
         self.write(s, escape=False)
 
     def include_files(self, extensions=None):
-
         """
         Includes files (usually in the head).
         Can minify and cache local files
@@ -465,46 +473,67 @@ class Response(Storage):
         response.cache_includes = (cache_method, time_expire).
         Example: (cache.disk, 60) # caches to disk for 1 minute.
         """
+        app = current.request.application
+
+        # We start by building a files list in which adjacent files internal to
+        # the application are placed in a list inside the files list.
+        #
+        # We will only minify and concat adjacent internal files as there's
+        # no way to know if changing the order with which the files are apppended
+        # will break things since the order matters in both CSS and JS and 
+        # internal files may be interleaved with external ones.
         files = []
-        ext_files = []
-        has_js = has_css = False
+        # For the adjacent list we're going to use storage List to both distinguish
+        # from the regular list and so we can add attributes
+        internal = List()  
+        internal.has_js = False
+        internal.has_css = False
+        done = set() # to remove duplicates
         for item in self.files:
-            if isinstance(item, (list, tuple)):
-                ext_files.append(item)
+            if not isinstance(item, list):
+                if item in done:
+                    continue
+                done.add(item)
+            if isinstance(item, (list, tuple)) or not item.startswith('/' + app): # also consider items in other web2py applications to be external
+                if internal:
+                    files.append(internal)
+                    internal = List()
+                    internal.has_js = False
+                    internal.has_css = False
+                files.append(item)
                 continue
             if extensions and not item.rpartition('.')[2] in extensions:
                 continue
-            if item in files:
-                continue
+            internal.append(item)
             if item.endswith('.js'):
-                has_js = True
+                internal.has_js = True
             if item.endswith('.css'):
-                has_css = True
-            files.append(item)
+                internal.has_css = True            
+        if internal:
+            files.append(internal)
 
-        if have_minify and ((self.optimize_css and has_css) or (self.optimize_js and has_js)):
-            # cache for 5 minutes by default
-            key = hashlib.md5(repr(files)).hexdigest()
-
-            cache = self.cache_includes or (current.cache.ram, 60 * 5)
-
-            def call_minify(files=files):
-                return minify.minify(files,
-                                     URL('static', 'temp'),
-                                     current.request.folder,
-                                     self.optimize_css,
-                                     self.optimize_js)
-            if cache:
-                cache_model, time_expire = cache
-                files = cache_model('response.files.minified/' + key,
-                                    call_minify,
-                                    time_expire)
-            else:
-                files = call_minify()
-
-        files.extend(ext_files)
-        s = []
-        for item in files:
+        # We're done we can now minify
+        if have_minify:
+            for i, f in enumerate(files):
+                if isinstance(f, List) and ((self.optimize_css and f.has_css) or (self.optimize_js and f.has_js)):
+                    # cache for 5 minutes by default
+                    key = hashlib_md5(repr(f)).hexdigest()
+                    cache = self.cache_includes or (current.cache.ram, 60 * 5)
+                    def call_minify(files=f):
+                        return List(minify.minify(files,
+                                             URL('static', 'temp'),
+                                             current.request.folder,
+                                             self.optimize_css,
+                                             self.optimize_js))
+                    if cache:
+                        cache_model, time_expire = cache
+                        files[i] = cache_model('response.files.minified/' + key,
+                                            call_minify,
+                                            time_expire)
+                    else:
+                        files[i] = call_minify()
+        
+        def static_map(s, item):
             if isinstance(item, str):
                 f = item.lower().split('?')[0]
                 ext = f.rpartition('.')[2]
@@ -523,6 +552,14 @@ class Response(Storage):
                 tmpl = template_mapping.get(f)
                 if tmpl:
                     s.append(tmpl % item[1])
+
+        s = []
+        for item in files:
+            if isinstance(item, List):
+                for f in item:
+                    static_map(s, f)
+            else:
+                static_map(s, item)
         self.write(''.join(s), escape=False)
 
     def stream(self,
@@ -578,9 +615,9 @@ class Response(Storage):
         if hasattr(stream, 'name'):
             filename = stream.name
 
-        if filename and not 'content-type' in keys:
+        if filename and 'content-type' not in keys:
             headers['Content-Type'] = contenttype(filename)
-        if filename and not 'content-length' in keys:
+        if filename and 'content-length' not in keys:
             try:
                 headers['Content-Length'] = \
                     os.path.getsize(filename)
@@ -638,6 +675,11 @@ class Response(Storage):
         if download_filename is None:
             download_filename = filename
         if attachment:
+            # Browsers still don't have a simple uniform way to have non ascii
+            # characters in the filename so for now we are percent encoding it
+            if isinstance(download_filename, unicodeT):
+                download_filename = download_filename.encode('utf-8')
+            download_filename = urllib_quote(download_filename)
             headers['Content-Disposition'] = \
                 'attachment; filename="%s"' % download_filename.replace('"', '\"')
         return self.stream(stream, chunk_size=chunk_size, request=request)
@@ -1022,7 +1064,7 @@ class Session(Storage):
         if self._forget:
             del rcookies[response.session_id_name]
             return
-        if self.get('httponly_cookies',True):
+        if self.get('httponly_cookies', True):
             scookies['HttpOnly'] = True
         if self._secure:
             scookies['secure'] = True
@@ -1193,7 +1235,7 @@ class Session(Storage):
             if (not response.session_id or
                 not response.session_filename or
                 self._forget
-                or self._unchanged(response)):
+                    or self._unchanged(response)):
                 # self.clear_session_cookies()
                 return False
             else:

gluon/highlight.py

@@ -182,8 +182,8 @@ class Highlighter(object):
         )),
         'PYTHONMultilineString': (python_tokenizer,
                                   (('ENDMULTILINESTRING',
-                                  re.compile(r'.*?("""|\'\'\')',
-                                  re.DOTALL), 'color: darkred'), )),
+                                    re.compile(r'.*?("""|\'\'\')',
+                                               re.DOTALL), 'color: darkred'), )),
         'HTML': (html_tokenizer, (
             ('GOTOPYTHON', re.compile(r'\{\{'), 'color: red'),
             ('COMMENT', re.compile(r'<!--[^>]*-->|<!>'),
@@ -209,7 +209,7 @@ class Highlighter(object):
         mode = self.mode
         while i < len(data):
             for (token, o_re, style) in Highlighter.all_styles[mode][1]:
-                if not token in self.suppress_tokens:
+                if token not in self.suppress_tokens:
                     match = o_re.match(data, i)
                     if match:
                         if style:
@@ -221,7 +221,7 @@ class Highlighter(object):
                             new_mode = \
                                 Highlighter.all_styles[mode][0](self,
                                                                 token, match, style)
-                        if not new_mode is None:
+                        if new_mode is not None:
                             mode = new_mode
                         i += max(1, len(match.group()))
                         break
@@ -241,9 +241,9 @@ class Highlighter(object):
             style = self.styles[token]
         if self.span_style != style:
             if style != 'Keep':
-                if not self.span_style is None:
+                if self.span_style is not None:
                     self.output.append('</span>')
-                if not style is None:
+                if style is not None:
                     self.output.append('<span style="%s">' % style)
                 self.span_style = style
 
@@ -260,7 +260,7 @@ def highlight(
 ):
     styles = styles or {}
     attributes = attributes or {}
-    if not 'CODE' in styles:
+    if 'CODE' not in styles:
         code_style = """
         font-size: 11px;
         font-family: Bitstream Vera Sans Mono,monospace;
@@ -272,7 +272,7 @@ def highlight(
         white-space: pre !important;\n"""
     else:
         code_style = styles['CODE']
-    if not 'LINENUMBERS' in styles:
+    if 'LINENUMBERS' not in styles:
         linenumbers_style = """
         font-size: 11px;
         font-family: Bitstream Vera Sans Mono,monospace;
@@ -283,7 +283,7 @@ def highlight(
         color: #A0A0A0;\n"""
     else:
         linenumbers_style = styles['LINENUMBERS']
-    if not 'LINEHIGHLIGHT' in styles:
+    if 'LINEHIGHLIGHT' not in styles:
         linehighlight_style = "background-color: #EBDDE2;"
     else:
         linehighlight_style = styles['LINEHIGHLIGHT']
@@ -333,8 +333,9 @@ def highlight(
                   == '_' and value])
     if fa:
         fa = ' ' + fa
-    return '<table%s><tr style="vertical-align:top;"><td style="min-width:40px; text-align: right;"><pre style="%s">%s</pre></td><td><pre style="%s">%s</pre></td></tr></table>'\
-        % (fa, linenumbers_style, numbers, code_style, code)
+    return '<table%s><tr style="vertical-align:top;">' \
+           '<td style="min-width:40px; text-align: right;"><pre style="%s">%s</pre></td>' \
+           '<td><pre style="%s">%s</pre></td></tr></table>' % (fa, linenumbers_style, numbers, code_style, code)
 
 
 if __name__ == '__main__':
@@ -342,5 +343,4 @@ if __name__ == '__main__':
     argfp = open(sys.argv[1])
     data = argfp.read()
     argfp.close()
-    print('<html><body>' + highlight(data, sys.argv[2])\
-        + '</body></html>')
+    print('<html><body>' + highlight(data, sys.argv[2]) + '</body></html>')

gluon/html.py

@@ -20,7 +20,8 @@ import urllib
 import base64
 from gluon import sanitizer, decoder
 import itertools
-from gluon._compat import reduce, pickle, copyreg, HTMLParser, name2codepoint, iteritems, unichr, unicodeT, urllib_quote, to_bytes, to_native, to_unicode, basestring, urlencode, implements_bool, text_type, long
+from gluon._compat import reduce, pickle, copyreg, HTMLParser, name2codepoint, iteritems, unichr, unicodeT, \
+    urllib_quote, to_bytes, to_native, to_unicode, basestring, urlencode, implements_bool, text_type, long
 from gluon.utils import local_html_escape
 import marshal
 
@@ -109,6 +110,7 @@ __all__ = [
 
 DEFAULT_PASSWORD_DISPLAY = '*' * 8
 
+
 def xmlescape(data, quote=True):
     """
     Returns an escaped string of the provided data
@@ -124,10 +126,9 @@ def xmlescape(data, quote=True):
 
     if not(isinstance(data, (text_type, bytes))):
         # i.e., integers
-        data=str(data)
+        data = str(data)
     data = to_bytes(data, 'utf8', 'xmlcharrefreplace')
 
-
     # ... and do the escaping
     data = local_html_escape(data, quote)
     return data
@@ -671,6 +672,7 @@ def XML_pickle(data):
     return XML_unpickle, (marshal.dumps(str(data)),)
 copyreg.pickle(XML, XML_pickle, XML_unpickle)
 
+
 @implements_bool
 class DIV(XmlComponent):
     """
@@ -1309,8 +1311,10 @@ class HTML(DIV):
     tag = b'html'
 
     strict = b'<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">\n'
-    transitional = b'<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">\n'
-    frameset = b'<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Frameset//EN" "http://www.w3.org/TR/html4/frameset.dtd">\n'
+    transitional = \
+        b'<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">\n'
+    frameset = \
+        b'<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Frameset//EN" "http://www.w3.org/TR/html4/frameset.dtd">\n'
     html5 = b'<!DOCTYPE HTML>\n'
 
     def xml(self):
@@ -1861,7 +1865,7 @@ class INPUT(DIV):
                 except:
                     import traceback
                     print(traceback.format_exc())
-                    msg = "Validation error, field:%s %s" % (name,validator)
+                    msg = "Validation error, field:%s %s" % (name, validator)
                     raise Exception(msg)
                 if errors is not None:
                     self.vars[name] = value
@@ -1911,7 +1915,7 @@ class INPUT(DIV):
         name = self.attributes.get('_name', None)
         if name and hasattr(self, 'errors') \
                 and self.errors.get(name, None) \
-                and self['hideerror'] != True:
+                and self['hideerror'] is not True:
             self['_class'] = (self['_class'] and self['_class'] + ' ' or '') + 'invalidinput'
             return DIV.xml(self) + DIV(
                 DIV(
@@ -1979,7 +1983,6 @@ class OPTGROUP(DIV):
 
 
 class SELECT(INPUT):
-
     """
     Examples:
 
@@ -2014,7 +2017,7 @@ class SELECT(INPUT):
         if value is not None:
             if not self['_multiple']:
                 for c in options:  # my patch
-                    if ((value is not None) and (str(c['_value']) == str(value))):
+                    if (value is not None) and (str(c['_value']) == str(value)):
                         c['_selected'] = 'selected'
                     else:
                         c['_selected'] = None
@@ -2024,7 +2027,7 @@ class SELECT(INPUT):
                 else:
                     values = [str(value)]
                 for c in options:  # my patch
-                    if ((value is not None) and (str(c['_value']) in values)):
+                    if (value is not None) and (str(c['_value']) in values):
                         c['_selected'] = 'selected'
                     else:
                         c['_selected'] = None
@@ -2390,7 +2393,6 @@ class FORM(DIV):
 
 
 class BEAUTIFY(DIV):
-
     """
     Turns any list, dictionary, etc into decent looking html.
 
@@ -2429,7 +2431,7 @@ class BEAUTIFY(DIV):
         if level == 0:
             return
         for c in self.components:
-            if hasattr(c, 'value') and not callable(c.value):
+            if hasattr(c, 'value') and not callable(c.value) and not isinstance(c, cgi.FieldStorage):
                 if c.value:
                     components.append(c.value)
             if hasattr(c, 'xml') and callable(c.xml):
@@ -2547,7 +2549,7 @@ class MENU(DIV):
                         li['_class'] = li['_class'] + ' ' + self['li_active']
                     else:
                         li['_class'] = self['li_active']
-                if len(item) <= 4 or item[4] == True:
+                if len(item) <= 4 or item[4] is True:
                     ul.append(li)
         return ul
 
@@ -2561,7 +2563,7 @@ class MENU(DIV):
                 # ex: ('', False, A('title', _href=URL(...), _title="title"))
                 # ex: (A('title', _href=URL(...), _title="title"), False, None)
                 custom_items.append(item)
-            elif len(item) <= 4 or item[4] == True:
+            elif len(item) <= 4 or item[4] is True:
                 select.append(OPTION(CAT(prefix, item[0]),
                                      _value=item[2], _selected=item[1]))
                 if len(item) > 3 and len(item[3]):
@@ -2703,7 +2705,8 @@ class web2pyHTMLParser(HTMLParser):
                 self.parent = self.parent.parent
             except:
                 raise RuntimeError("unable to balance tag %s" % tagname)
-            if parent_tagname[:len(tagname)] == tagname: break
+            if parent_tagname[:len(tagname)] == tagname:
+                break
 
 
 def markdown_serializer(text, tag=None, attr=None):

gluon/http.py

@@ -11,7 +11,7 @@ HTTP statuses helpers
 """
 
 import re
-from gluon._compat import iteritems
+from gluon._compat import iteritems, unicodeT, to_bytes
 
 __all__ = ['HTTP', 'redirect']
 
@@ -111,22 +111,29 @@ class HTTP(Exception):
             if not body:
                 body = status
             if isinstance(body, (str, bytes, bytearray)):
+                if isinstance(body, unicodeT):
+                    body = to_bytes(body) # This must be done before len
                 headers['Content-Length'] = len(body)
         rheaders = []
         for k, v in iteritems(headers):
             if isinstance(v, list):
                 rheaders += [(k, str(item)) for item in v]
-            elif not v is None:
+            elif v is not None:
                 rheaders.append((k, str(v)))
         responder(status, rheaders)
         if env.get('request_method', '') == 'HEAD':
             return ['']
         elif isinstance(body, (str, bytes, bytearray)):
+            if isinstance(body, unicodeT):
+                body = to_bytes(body)
             return [body]
         elif hasattr(body, '__iter__'):
             return body
         else:
-            return [str(body)]
+            body = str(body)
+            if isinstance(body, unicodeT):
+                body = to_bytes(body)
+            return [body]
 
     @property
     def message(self):
@@ -148,7 +155,7 @@ class HTTP(Exception):
             web2py_error=self.headers.get('web2py_error'))
 
     def __str__(self):
-        "stringify me"
+        """stringify me"""
         return self.message
 
 

gluon/import_all.py

@@ -78,13 +78,9 @@ alert_dependency = ['hashlib', 'uuid']
 # Now we remove the blacklisted modules if we are using the stated
 # python version.
 #
-# List of modules deprecated in Python 2.6 or 2.7 that are in the above set
+# List of modules deprecated in Python 2.7 that are in the above list
 py27_deprecated = ['mhlib', 'multifile', 'mimify', 'sets', 'MimeWriter']  # And ['optparse'] but we need it for now
 
-if python_version >= '2.6':
-    base_modules += ['json', 'multiprocessing']
-    base_modules = list(set(base_modules).difference(set(py26_deprecated)))
-
 if python_version >= '2.7':
     base_modules += ['argparse', 'json', 'multiprocessing']
     base_modules = list(set(base_modules).difference(set(py27_deprecated)))

gluon/main.py

@@ -11,7 +11,8 @@ The gluon wsgi application
 """
 from __future__ import print_function
 
-if False: import import_all # DO NOT REMOVE PART OF FREEZE PROCESS
+if False:
+    import import_all  # DO NOT REMOVE PART OF FREEZE PROCESS
 import gc
 
 import os
@@ -26,7 +27,7 @@ import random
 import string
 
 from gluon._compat import Cookie, urllib2
-#from thread import allocate_lock
+# from thread import allocate_lock
 
 from gluon.fileutils import abspath, write_file
 from gluon.settings import global_settings
@@ -67,14 +68,14 @@ import gluon.messageboxhandler
 logging.gluon = gluon
 # so we must restore it! Thanks ozancag
 import locale
-locale.setlocale(locale.LC_CTYPE, "C") # IMPORTANT, web2py requires locale "C"
+locale.setlocale(locale.LC_CTYPE, "C")  # IMPORTANT, web2py requires locale "C"
 
 exists = os.path.exists
 pjoin = os.path.join
 
 try:
     logging.config.fileConfig(abspath("logging.conf"))
-except: # fails on GAE or when logfile is missing
+except:  # fails on GAE or when logfile is missing
     logging.basicConfig()
 logger = logging.getLogger("web2py")
 
@@ -254,6 +255,7 @@ class LazyWSGI(object):
                 return [data]
             for item in middleware_apps:
                 app = item(app)
+
             def caller(app):
                 return app(self.environ, self.start_response)
             return lambda caller=caller, app=app: caller(app)
@@ -294,9 +296,9 @@ def wsgibase(environ, responder):
     response = Response()
     session = Session()
     env = request.env
-    #env.web2py_path = global_settings.applications_parent
+    # env.web2py_path = global_settings.applications_parent
     env.web2py_version = web2py_version
-    #env.update(global_settings)
+    # env.update(global_settings)
     static_file = False
     http_response = None
     try:
@@ -325,7 +327,6 @@ def wsgibase(environ, responder):
                             'Expires'] = 'Thu, 31 Dec 2037 23:59:59 GMT'
                     response.stream(static_file, request=request)
 
-
                 # ##################################################
                 # fill in request items
                 # ##################################################
@@ -356,17 +357,15 @@ def wsgibase(environ, responder):
                 cmd_opts = global_settings.cmd_options
 
                 request.update(
-                    client = client,
-                    folder = abspath('applications', app) + os.sep,
-                    ajax = x_req_with == 'xmlhttprequest',
-                    cid = env.http_web2py_component_element,
-                    is_local = (env.remote_addr in local_hosts and
-                                client == env.remote_addr),
-                    is_shell = False,
-                    is_scheduler = False,
-                    is_https = env.wsgi_url_scheme in HTTPS_SCHEMES or \
-                        request.env.http_x_forwarded_proto in HTTPS_SCHEMES \
-                        or env.https == 'on'
+                    client=client,
+                    folder=abspath('applications', app) + os.sep,
+                    ajax=x_req_with == 'xmlhttprequest',
+                    cid=env.http_web2py_component_element,
+                    is_local=(env.remote_addr in local_hosts and client == env.remote_addr),
+                    is_shell=False,
+                    is_scheduler=False,
+                    is_https=env.wsgi_url_scheme in HTTPS_SCHEMES or
+                             request.env.http_x_forwarded_proto in HTTPS_SCHEMES or env.https == 'on'
                     )
                 request.url = environ['PATH_INFO']
 
@@ -390,7 +389,7 @@ def wsgibase(environ, responder):
                                    % 'invalid request',
                                    web2py_error='invalid application')
                 elif not request.is_local and exists(disabled):
-                    five0three = os.path.join(request.folder,'static','503.html')
+                    five0three = os.path.join(request.folder, 'static', '503.html')
                     if os.path.exists(five0three):
                         raise HTTP(503, file(five0three, 'r').read())
                     else:
@@ -406,7 +405,7 @@ def wsgibase(environ, responder):
                 # get the GET and POST data
                 # ##################################################
 
-                #parse_get_post_vars(request, environ)
+                # parse_get_post_vars(request, environ)
 
                 # ##################################################
                 # expose wsgi hooks for convenience
@@ -625,7 +624,7 @@ def appfactory(wsgiapp=wsgibase,
                 raise BaseException("Can't create dir %s" % profiler_dir)
         filepath = pjoin(profiler_dir, 'wtest')
         try:
-            filehandle = open( filepath, 'w' )
+            filehandle = open(filepath, 'w')
             filehandle.close()
             os.unlink(filepath)
         except IOError:
@@ -746,7 +745,7 @@ class HttpServer(object):
         sock_list = [ip, port]
         if not ssl_certificate or not ssl_private_key:
             logger.info('SSL is off')
-        elif not rocket.ssl:
+        elif not rocket.has_ssl:
             logger.warning('Python "ssl" module unavailable. SSL is OFF')
         elif not exists(ssl_certificate):
             logger.warning('unable to open SSL certificate. SSL is OFF')

gluon/newcron.py

@@ -225,7 +225,7 @@ def parsecronline(line):
     params = line.strip().split(None, 6)
     if len(params) < 7:
         return None
-    daysofweek = {'sun': 0, 'mon': 1, 'tue': 2, 'wed': 3, 
+    daysofweek = {'sun': 0, 'mon': 1, 'tue': 2, 'wed': 3,
                   'thu': 4, 'fri': 5, 'sat': 6}
     for (s, id) in zip(params[:5], ['min', 'hr', 'dom', 'mon', 'dow']):
         if not s in [None, '*']:

gluon/restricted.py

@@ -10,7 +10,7 @@ Restricted environment to execute application's code
 """
 
 import sys
-from gluon._compat import pickle, ClassType
+from gluon._compat import pickle, ClassType, unicodeT, to_bytes
 import traceback
 import types
 import os
@@ -192,10 +192,10 @@ class RestrictedError(Exception):
         # safely show an useful message to the user
         try:
             output = self.output
-            if isinstance(output, unicode):
-                output = output.encode("utf8")
-            elif not isinstance(output, str):
+            if not isinstance(output, str, bytes, bytearray):
                 output = str(output)
+            if isinstance(output, unicodeT):
+                output = to_bytes(output)
         except:
             output = ""
         return output
@@ -205,7 +205,7 @@ def compile2(code, layer):
     return compile(code, layer, 'exec')
 
 
-def restricted(ccode, environment=None, layer='Unknown'):
+def restricted(ccode, environment=None, layer='Unknown', scode=None):
     """
     Runs code in environment and returns the output. If an exception occurs
     in code it raises a RestrictedError containing the traceback. Layer is
@@ -230,7 +230,9 @@ def restricted(ccode, environment=None, layer='Unknown'):
             sys.excepthook(etype, evalue, tb)
         del tb
         output = "%s %s" % (etype, evalue)
-        raise RestrictedError(layer, ccode, output, environment)
+        # Save source code in ticket when available
+        scode = scode if scode else ccode
+        raise RestrictedError(layer, scode, output, environment)
 
 
 def snapshot(info=None, context=5, code=None, environment=None):

gluon/rewrite.py

@@ -206,8 +206,7 @@ def url_out(request, environ, application, controller, function,
     if host is True or (host is None and (scheme or port is not None)):
         host = request.env.http_host
     if not scheme or scheme is True:
-        scheme = request.env.get('wsgi_url_scheme', 'http').lower() \
-            if request else 'http'
+        scheme = request.env.get('wsgi_url_scheme', 'http').lower() if request else 'http'
     if host:
         host_port = host if not port else host.split(':', 1)[0] + ':%s' % port
         url = '%s://%s%s' % (scheme, host_port, url)

gluon/sanitizer.py

@@ -145,7 +145,7 @@ class XssCleaner(HTMLParser):
         if url.startswith('#'):
             return True
         else:
-            parsed = urlparse(url)
+            parsed = urlparse.urlparse(url)
             return ((parsed[0] in self.allowed_schemes and '.' in parsed[1]) or
                     (parsed[0] in self.allowed_schemes and '@' in parsed[2]) or
                     (parsed[0] == '' and parsed[2].startswith('/')))

gluon/serializers.py

@@ -119,8 +119,8 @@ def xml(value, encoding='UTF-8', key='document', quote=True):
     return ('<?xml version="1.0" encoding="%s"?>' % encoding) + str(xml_rec(value, key, quote))
 
 
-def json(value, default=custom_json, indent=None):
-    value = json_parser.dumps(value, default=default, sort_keys=True, indent=indent)
+def json(value, default=custom_json, indent=None, sort_keys=False):
+    value = json_parser.dumps(value, default=default, sort_keys=sort_keys, indent=indent)
     # replace JavaScript incompatible spacing
     # http://timelessrepo.com/json-isnt-a-javascript-subset
     # PY3 FIXME

gluon/shell.py

@@ -31,10 +31,16 @@ from gluon.globals import Request, Response, Session
 from gluon.storage import Storage, List
 from gluon.admin import w2p_unpack
 from pydal.base import BaseAdapter
-from gluon._compat import iteritems, ClassType
+from gluon._compat import iteritems, ClassType, PY2
 
 logger = logging.getLogger("web2py")
 
+if not PY2:
+    def execfile(filename, global_vars=None, local_vars=None):
+        with open(filename) as f:
+            code = compile(f.read(), filename, 'exec')
+            exec(code, global_vars, local_vars)
+
 
 def enable_autocomplete_and_history(adir, env):
     try:

gluon/sqlhtml.py

@@ -405,7 +405,7 @@ class RadioWidget(OptionsWidget):
         cols = attributes.get('cols', 1)
         totals = len(options)
         mods = totals % cols
-        rows = totals / cols
+        rows = totals // cols
         if mods:
             rows += 1
 
@@ -471,7 +471,7 @@ class CheckboxesWidget(OptionsWidget):
         cols = attributes.get('cols', 1)
         totals = len(options)
         mods = totals % cols
-        rows = totals / cols
+        rows = totals // cols
         if mods:
             rows += 1
 
@@ -658,7 +658,8 @@ class AutocompleteWidget(object):
                  orderby=None, limitby=(0, 10), distinct=False,
                  keyword='_autocomplete_%(tablename)s_%(fieldname)s',
                  min_length=2, help_fields=None, help_string=None,
-                 at_beginning=True, default_var='ac'):
+                 at_beginning=True, default_var='ac', user_signature=True,
+                 hash_vars=False):
 
         self.help_fields = help_fields or []
         self.help_string = help_string
@@ -683,10 +684,12 @@ class AutocompleteWidget(object):
         if hasattr(request, 'application'):
             urlvars = request.vars
             urlvars[default_var] = 1
-            self.url = URL(args=request.args, vars=urlvars)
-            self.callback()
+            self.url = URL(args=request.args, vars=urlvars,
+                           user_signature=user_signature, hash_vars=hash_vars)
+            self.run_callback = True
         else:
             self.url = request
+            self.run_callback = False
 
     def callback(self):
         if self.keyword in self.request.vars:
@@ -759,6 +762,8 @@ class AutocompleteWidget(object):
                 raise HTTP(200, '')
 
     def __call__(self, field, value, **attributes):
+        if self.run_callback:
+            self.callback()
         default = dict(
             _type='text',
             value=(value is not None and str(value)) or '',
@@ -1916,8 +1921,10 @@ class SQLFORM(FORM):
         if 'table_name' in attributes:
             del attributes['table_name']
 
-        return SQLFORM(DAL(None).define_table(table_name, *fields),
-                       **attributes)
+        # Clone fields, while passing tables straight through
+        fields_with_clones = [f.clone() if isinstance(f, Field) else f for f in fields]
+
+        return SQLFORM(DAL(None).define_table(table_name, *fields_with_clones), **attributes)
 
     @staticmethod
     def build_query(fields, keywords):
@@ -1932,11 +1939,13 @@ class SQLFORM(FORM):
             if settings.global_settings.web2py_runtime_gae:
                 return reduce(lambda a,b: a|b, [field.contains(key) for field in sfields])
             else:
+                if not (sfields and key and key.split()):
+                    return fields[0].table
                 return reduce(lambda a,b:a&b,[
                         reduce(lambda a,b: a|b, [
                                 field.contains(k) for field in sfields]
                                ) for k in key.split()])
-
+                    
             # from https://groups.google.com/forum/#!topic/web2py/hKe6lI25Bv4
             # needs testing...
             #words = key.split(' ') if key else []
@@ -2156,6 +2165,7 @@ class SQLFORM(FORM):
              represent_none=None,
              showblobs=False):
 
+        dbset = None
         formstyle = formstyle or current.response.formstyle
         if isinstance(query, Set):
             query = query.query
@@ -3034,6 +3044,7 @@ class SQLFORM(FORM):
         res.view_form = view_form
         res.search_form = search_form
         res.rows = rows
+        res.dbset = dbset
         return res
 
     @staticmethod
@@ -3152,8 +3163,8 @@ class SQLFORM(FORM):
                 # if isinstance(linked_tables, dict):
                 #     linked_tables = linked_tables.get(table._tablename, [])
                 if linked_tables is None or referee in linked_tables:
-                    field.represent = (lambda id, r=None, referee=referee, rep=field.represent: 
-                                       A(callable(rep) and rep(id) or id, 
+                    field.represent = (lambda id, r=None, referee=referee, rep=field.represent:
+                                       A(callable(rep) and rep(id) or id,
                                          cid=request.cid, _href=url(args=['view', referee, id])))
         except (KeyError, ValueError, TypeError):
             redirect(URL(args=table._tablename))

gluon/tests/test_appadmin.py

@@ -17,6 +17,7 @@ from gluon import fileutils
 from gluon.dal import DAL, Field, Table
 from gluon.http import HTTP
 from gluon.fileutils import open_file
+from gluon.cache import CacheInRam
 
 DEFAULT_URI = os.getenv('DB', 'sqlite:memory')
 
@@ -104,6 +105,21 @@ class TestAppAdmin(unittest.TestCase):
         self._test_index()
         remove_compiled_application(appname_path)
 
+    def test_index_minify(self):
+        # test for gluon/contrib/minify
+        self.env['response'].optimize_css = 'concat|minify'
+        self.env['response'].optimize_js = 'concat|minify'
+        self.env['current'].cache = Storage({'ram':CacheInRam()})
+        appname_path = os.path.join(os.getcwd(), 'applications', 'welcome')
+        self._test_index()
+        file_l = os.listdir(os.path.join(appname_path, 'static', 'temp'))
+        file_l.sort()
+        self.assertTrue(len(file_l) == 2)
+        self.assertEqual(file_l[0][0:10], 'compressed')
+        self.assertEqual(file_l[1][0:10], 'compressed')
+        self.assertEqual(file_l[0][-3:], 'css')
+        self.assertEqual(file_l[1][-2:], 'js')
+
     def test_select(self):
         request = self.env['request']
         request.args = List(['db'])

gluon/tests/test_fileutils.py

@@ -1,10 +1,11 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
+import os
 import unittest
 import datetime
 
-from gluon.fileutils import parse_version
+from gluon.fileutils import parse_version, fix_newlines
 
 
 class TestFileUtils(unittest.TestCase):
@@ -22,3 +23,6 @@ class TestFileUtils(unittest.TestCase):
         # Semantic Beta
         rtn = parse_version('Version 2.14.1-beta+timestamp.2016.03.21.22.35.26')
         self.assertEqual(rtn, (2, 14, 1, 'beta', datetime.datetime(2016, 3, 21, 22, 35, 26)))
+    
+    def test_fix_newlines(self):
+        fix_newlines(os.path.dirname(os.path.abspath(__file__)))

gluon/tests/test_globals.py

@@ -158,10 +158,10 @@ class testResponse(unittest.TestCase):
         response.files.append(URL('a', 'static', 'css/file.ts'))
         content = return_includes(response)
         self.assertEqual(content,
+                         '<script src="http://maps.google.com/maps/api/js?sensor=false" type="text/javascript"></script>' +
                          '<script src="https://code.jquery.com/jquery-1.11.3.min.js?var=0" type="text/javascript"></script>' +
                          '<link href="/a/static/css/file.css" rel="stylesheet" type="text/css" />' +
-                         '<script src="/a/static/css/file.ts" type="text/typescript"></script>' +
-                         '<script src="http://maps.google.com/maps/api/js?sensor=false" type="text/javascript"></script>'
+                         '<script src="/a/static/css/file.ts" type="text/typescript"></script>'
                          )
 
         response = Response()

gluon/tools.py

@@ -1127,7 +1127,6 @@ def addrow(form, a, b, c, style, _id, position=-1):
 
 
 class AuthJWT(object):
-
     """
     Experimental!
 
@@ -1321,7 +1320,7 @@ class AuthJWT(object):
         # is the following safe or should we use
         # calendar.timegm(datetime.datetime.utcnow().timetuple())
         # result seem to be the same (seconds since epoch, in UTC)
-        now = time.mktime(datetime.datetime.now().timetuple())
+        now = time.mktime(datetime.datetime.utcnow().timetuple())
         expires = now + self.expiration
         payload = dict(
             hmac_key=session_auth['hmac_key'],
@@ -1333,7 +1332,7 @@ class AuthJWT(object):
         return payload
 
     def refresh_token(self, orig_payload):
-        now = time.mktime(datetime.datetime.now().timetuple())
+        now = time.mktime(datetime.datetime.utcnow().timetuple())
         if self.verify_expiration:
             orig_exp = orig_payload['exp']
             if orig_exp + self.leeway < now:
@@ -1537,7 +1536,8 @@ class Auth(AuthAPI):
     # ## these are messages that can be customized
     default_messages = dict(AuthAPI.default_messages,
                             access_denied='Insufficient privileges',
-                            bulk_invite_body='You have been invited to join %(site)s, click %(link)s to complete the process',
+                            bulk_invite_body='You have been invited to join %(site)s, click %(link)s to complete '
+                                             'the process',
                             bulk_invite_subject='Invitation to join %(site)s',
                             delete_label='Check to delete',
                             email_sent='Email sent',
@@ -1762,7 +1762,7 @@ class Auth(AuthAPI):
             if auth.last_visit and auth.last_visit + delta > now:
                 self.user = auth.user
                 # this is a trick to speed up sessions to avoid many writes
-                if (now - auth.last_visit).seconds > (auth.expiration / 10):
+                if (now - auth.last_visit).seconds > (auth.expiration // 10):
                     auth.last_visit = now
             else:
                 self.user = None
@@ -1792,6 +1792,7 @@ class Auth(AuthAPI):
                              servicevalidate='serviceValidate',
                              proxyvalidate='proxyValidate',
                              logout='logout'),
+            cas_create_user=True,
             extra_fields={},
             actions_disabled=[],
             controller=controller,
@@ -1840,14 +1841,15 @@ class Auth(AuthAPI):
         # ## these are messages that can be customized
         messages = self.messages = Messages(current.T)
         messages.update(Auth.default_messages)
-        messages.update(ajax_failed_authentication=DIV(H4('NOT AUTHORIZED'),
-                                                       'Please ',
-                                                       A('login',
-                                                         _href=self.settings.login_url +
-                                                         ('?_next=' + urllib_quote(current.request.env.http_web2py_component_location))
-                                                         if current.request.env.http_web2py_component_location else ''),
-                                                       ' to view this content.',
-                                                       _class='not-authorized alert alert-block'))
+        messages.update(ajax_failed_authentication=
+                        DIV(H4('NOT AUTHORIZED'),
+                            'Please ',
+                            A('login',
+                              _href=self.settings.login_url +
+                                    ('?_next=' + urllib_quote(current.request.env.http_web2py_component_location))
+                              if current.request.env.http_web2py_component_location else ''),
+                            ' to view this content.',
+                            _class='not-authorized alert alert-block'))
         messages.lock_keys = True
 
         # for "remember me" option
@@ -1876,7 +1878,7 @@ class Auth(AuthAPI):
         # _next variable in the request.
         if next:
             parts = next.split('/')
-            if not ':' in parts[0]:
+            if ':' not in parts[0]:
                 return next
             elif len(parts) > 2 and parts[0].endswith(':') and parts[1:3] == ['', host]:
                 return next
@@ -2008,8 +2010,7 @@ class Auth(AuthAPI):
                 items.append({'name': T('Lost password?'),
                               'href': href('request_reset_password'),
                               'icon': 'icon-lock'})
-            if (self.settings.use_username and not
-                    'retrieve_username' in self.settings.actions_disabled):
+            if self.settings.use_username and 'retrieve_username' not in self.settings.actions_disabled:
                 items.append({'name': T('Forgot username?'),
                               'href': href('retrieve_username'),
                               'icon': 'icon-edit'})
@@ -2181,9 +2182,7 @@ class Auth(AuthAPI):
             current_record.replace('_', ' ').title())
         for table in tables:
             fieldnames = table.fields()
-            if ('id' in fieldnames and
-                    'modified_on' in fieldnames and
-                    not current_record in fieldnames):
+            if 'id' in fieldnames and 'modified_on' in fieldnames and current_record not in fieldnames:
                 table._enable_record_versioning(archive_db=archive_db,
                                                 archive_name=archive_names,
                                                 current_record=current_record,
@@ -2213,7 +2212,8 @@ class Auth(AuthAPI):
             fake_migrate = db._fake_migrate
         settings = self.settings
         settings.enable_tokens = enable_tokens
-        signature_list = super(Auth, self).define_tables(username, signature, migrate, fake_migrate)._table_signature_list
+        signature_list = \
+            super(Auth, self).define_tables(username, signature, migrate, fake_migrate)._table_signature_list
 
         now = current.request.now
         reference_table_user = 'reference %s' % settings.table_user_name
@@ -2285,6 +2285,7 @@ class Auth(AuthAPI):
         If the user doesn't yet exist, then they are created.
         """
         table_user = self.table_user()
+        create_user = self.settings.cas_create_user
         user = None
         checks = []
         # make a guess about who this user is
@@ -2317,6 +2318,11 @@ class Auth(AuthAPI):
                     update_keys[key] = keys[key]
             user.update_record(**update_keys)
         elif checks:
+            if create_user is False:
+                # Remove current open session a send message
+                self.logout(next=None, onlogout=None, log=None)
+                raise HTTP(403, "Forbidden. User need to be created first.")
+
             if 'first_name' not in keys and 'first_name' in table_user.fields:
                 guess = keys.get('email', 'anonymous').split('@')[0]
                 keys['first_name'] = keys.get('username', guess)
@@ -2360,7 +2366,7 @@ class Auth(AuthAPI):
             if callable(basic_auth_realm):
                 basic_auth_realm = basic_auth_realm()
             elif isinstance(basic_auth_realm, (unicode, str)):
-                basic_realm = unicode(basic_auth_realm)
+                basic_realm = unicode(basic_auth_realm)  # Warning python 3.5 does not have method unicod
             elif basic_auth_realm is True:
                 basic_realm = u'' + current.request.application
             http_401 = HTTP(401, u'Not Authorized', **{'WWW-Authenticate': u'Basic realm="' + basic_realm + '"'})
@@ -2462,7 +2468,7 @@ class Auth(AuthAPI):
                          _href=service + query_sep + "ticket=" + ticket)
             else:
                 redirect(service + query_sep + "ticket=" + ticket)
-        if self.is_logged_in() and not 'renew' in request.vars:
+        if self.is_logged_in() and 'renew' not in request.vars:
             return allow_access()
         elif not self.is_logged_in() and 'gateway' in request.vars:
             redirect(session._cas_service)
@@ -2779,7 +2785,7 @@ class Auth(AuthAPI):
         # If auth.settings.auth_two_factor_enabled it will enable two factor
         # for all the app. Another way to anble two factor is that the user
         # must be part of a group that is called auth.settings.two_factor_authentication_group
-        if user and self.settings.auth_two_factor_enabled == True:
+        if user and self.settings.auth_two_factor_enabled is True:
             session.auth_two_factor_enabled = True
         elif user and self.settings.two_factor_authentication_group:
             role = self.settings.two_factor_authentication_group
@@ -2809,7 +2815,7 @@ class Auth(AuthAPI):
                 # Set the way we generate the code or we send the code. For example using SMS...
                 two_factor_methods = self.settings.two_factor_methods
 
-                if two_factor_methods == []:
+                if not two_factor_methods:
                     # TODO: Add some error checking to handle cases where email cannot be sent
                     self.settings.mailer.send(
                         to=user.email,
@@ -2832,47 +2838,49 @@ class Auth(AuthAPI):
                             hideerror=settings.hideerror):
                 accepted_form = True
 
-                '''
+                """
                 The lists is executed after form validation for each of the corresponding action.
                 For example, in your model:
 
                 In your models copy and paste:
 
-                #Before define tables, we add some extra field to auth_user
+                # Before define tables, we add some extra field to auth_user
                 auth.settings.extra_fields['auth_user'] = [
                     Field('motp_secret', 'password', length=512, default='', label='MOTP Secret'),
                     Field('motp_pin', 'string', length=128, default='', label='MOTP PIN')]
 
-                OFFSET = 60 #Be sure is the same in your OTP Client
+                OFFSET = 60  # Be sure is the same in your OTP Client
 
-                #Set session.auth_two_factor to None. Because the code is generated by external app.
+                # Set session.auth_two_factor to None. Because the code is generated by external app.
                 # This will avoid to use the default setting and send a code by email.
                 def _set_two_factor(user, auth_two_factor):
                     return None
 
                 def verify_otp(user, otp):
-                import time
-                from hashlib import md5
-                epoch_time = int(time.time())
-                time_start = int(str(epoch_time - OFFSET)[:-1])
-                time_end = int(str(epoch_time + OFFSET)[:-1])
-                for t in range(time_start - 1, time_end + 1):
-                    to_hash = str(t) + user.motp_secret + user.motp_pin
-                    hash = md5(to_hash).hexdigest()[:6]
-                    if otp == hash:
-                    return hash
+                    import time
+                    from hashlib import md5
+                    epoch_time = int(time.time())
+                    time_start = int(str(epoch_time - OFFSET)[:-1])
+                    time_end = int(str(epoch_time + OFFSET)[:-1])
+                    for t in range(time_start - 1, time_end + 1):
+                        to_hash = str(t) + user.motp_secret + user.motp_pin
+                        hash = md5(to_hash).hexdigest()[:6]
+                        if otp == hash:
+                        return hash
 
                 auth.settings.auth_two_factor_enabled = True
                 auth.messages.two_factor_comment = "Verify your OTP Client for the code."
-                auth.settings.two_factor_methods = [lambda user, auth_two_factor: _set_two_factor(user, auth_two_factor)]
+                auth.settings.two_factor_methods = [lambda user,
+                                                           auth_two_factor: _set_two_factor(user, auth_two_factor)]
                 auth.settings.two_factor_onvalidation = [lambda user, otp: verify_otp(user, otp)]
 
-                '''
-                if self.settings.two_factor_onvalidation != []:
+                """
+                if self.settings.two_factor_onvalidation:
 
                     for two_factor_onvalidation in self.settings.two_factor_onvalidation:
                         try:
-                            session.auth_two_factor = two_factor_onvalidation(session.auth_two_factor_user, form.vars['authentication_code'])
+                            session.auth_two_factor = \
+                                two_factor_onvalidation(session.auth_two_factor_user, form.vars['authentication_code'])
                         except:
                             pass
                         else:
@@ -3655,6 +3663,16 @@ class Auth(AuthAPI):
         if not self.is_logged_in():
             redirect(self.settings.login_url,
                      client_side=self.settings.client_side)
+
+        # Go to external link to change the password
+        if self.settings.login_form != self:
+            cas = self.settings.login_form
+            # To prevent error if change_password_url function is not defined in alternate login
+            if hasattr(cas, 'change_password_url'):
+                next = cas.change_password_url(next)
+                if next is not None:
+                    redirect(next)
+
         db = self.db
         table_user = self.table_user()
         s = db(table_user.id == self.user.id)
@@ -4146,7 +4164,7 @@ class Auth(AuthAPI):
             archive_table = table._db[archive_table_name]
         new_record = {current_record: form.vars.id}
         for fieldname in archive_table.fields:
-            if not fieldname in ['id', current_record]:
+            if fieldname not in ['id', current_record]:
                 if archive_current and fieldname in form.vars:
                     new_record[fieldname] = form.vars[fieldname]
                 elif form.record and fieldname in form.record:
@@ -4956,7 +4974,10 @@ class Service(object):
 
             Then call it with:
 
-                wget --post-data '{"jsonrpc": "2.0", "id": 1, "method": "myfunction", "params": {"a": 1, "b": 2}}' http://..../app/default/call/jsonrpc2
+                wget --post-data '{"jsonrpc": "2.0",
+                                   "id": 1,
+                                   "method": "myfunction",
+                                   "params": {"a": 1, "b": 2}}' http://..../app/default/call/jsonrpc2
 
         """
         self.jsonrpc2_procedures[f.__name__] = f
@@ -5527,15 +5548,15 @@ def prettydate(d, T=lambda x: x, utc=False):
     else:
         suffix = ' ago'
     if dt.days >= 2 * 365:
-        return T('%d years' + suffix) % int(dt.days / 365)
+        return T('%d years' + suffix) % int(dt.days // 365)
     elif dt.days >= 365:
         return T('1 year' + suffix)
     elif dt.days >= 60:
-        return T('%d months' + suffix) % int(dt.days / 30)
+        return T('%d months' + suffix) % int(dt.days // 30)
     elif dt.days >= 27:  # 4 weeks ugly
         return T('1 month' + suffix)
     elif dt.days >= 14:
-        return T('%d weeks' + suffix) % int(dt.days / 7)
+        return T('%d weeks' + suffix) % int(dt.days // 7)
     elif dt.days >= 7:
         return T('1 week' + suffix)
     elif dt.days > 1:
@@ -5543,11 +5564,11 @@ def prettydate(d, T=lambda x: x, utc=False):
     elif dt.days == 1:
         return T('1 day' + suffix)
     elif dt.seconds >= 2 * 60 * 60:
-        return T('%d hours' + suffix) % int(dt.seconds / 3600)
+        return T('%d hours' + suffix) % int(dt.seconds // 3600)
     elif dt.seconds >= 60 * 60:
         return T('1 hour' + suffix)
     elif dt.seconds >= 2 * 60:
-        return T('%d minutes' + suffix) % int(dt.seconds / 60)
+        return T('%d minutes' + suffix) % int(dt.seconds // 60)
     elif dt.seconds >= 60:
         return T('1 minute' + suffix)
     elif dt.seconds > 1:
@@ -5600,12 +5621,12 @@ class PluginManager(object):
 
         where the plugin is used::
 
-            >>> print plugins.me.param1
+            >>> print(plugins.me.param1)
             3
-            >>> print plugins.me.param2
+            >>> print(plugins.me.param2)
             6
             >>> plugins.me.param3 = 8
-            >>> print plugins.me.param3
+            >>> print(plugins.me.param3)
             8
 
         Here are some tests::
@@ -5613,25 +5634,25 @@ class PluginManager(object):
             >>> a=PluginManager()
             >>> a.x=6
             >>> b=PluginManager('check')
-            >>> print b.x
+            >>> print(b.x)
             6
             >>> b=PluginManager() # reset settings
-            >>> print b.x
+            >>> print(b.x)
             <Storage {}>
             >>> b.x=7
-            >>> print a.x
+            >>> print(a.x)
             7
             >>> a.y.z=8
-            >>> print b.y.z
+            >>> print(b.y.z)
             8
             >>> test_thread_separation()
             5
             >>> plugins=PluginManager('me',db='mydb')
-            >>> print plugins.me.db
+            >>> print(plugins.me.db)
             mydb
-            >>> print 'me' in plugins
+            >>> print('me' in plugins)
             True
-            >>> print plugins.me.installed
+            >>> print(plugins.me.installed)
             True
 
     """
@@ -5780,7 +5801,7 @@ class Expose(object):
             return os.path.realpath(f)
 
     def issymlink_out(self, f):
-        "True if f is a symlink and is pointing outside of self.base"
+        """True if f is a symlink and is pointing outside of self.base"""
         return os.path.islink(f) and not self.in_base(f)
 
     @staticmethod

gluon/utils.py

@@ -156,12 +156,12 @@ def get_digest(value):
         raise ValueError("Invalid digest algorithm: %s" % value)
 
 DIGEST_ALG_BY_SIZE = {
-    128 / 4: 'md5',
-    160 / 4: 'sha1',
-    224 / 4: 'sha224',
-    256 / 4: 'sha256',
-    384 / 4: 'sha384',
-    512 / 4: 'sha512',
+    128 // 4: 'md5',
+    160 // 4: 'sha1',
+    224 // 4: 'sha224',
+    256 // 4: 'sha256',
+    384 // 4: 'sha384',
+    512 // 4: 'sha512',
 }
 
 

gluon/validators.py

@@ -21,7 +21,8 @@ import struct
 import decimal
 import unicodedata
 
-from gluon._compat import StringIO, long, basestring, unicodeT, to_unicode, urllib_unquote, unichr, to_bytes, PY2, to_unicode, to_native, string_types, urlparse
+from gluon._compat import StringIO, long, basestring, unicodeT, to_unicode, urllib_unquote, unichr, to_bytes, PY2, \
+    to_unicode, to_native, string_types, urlparse
 from gluon.utils import simple_hash, web2py_uuid, DIGEST_ALG_BY_SIZE
 from pydal.objects import Field, FieldVirtual, FieldMethod
 from functools import reduce
@@ -452,10 +453,10 @@ class IS_IN_SET(Validator):
         if not self.labels:
             items = [(k, k) for (i, k) in enumerate(self.theset)]
         else:
-            items = [(k, self.labels[i]) for (i, k) in enumerate(self.theset)]
+            items = [(k, list(self.labels)[i]) for (i, k) in enumerate(self.theset)]
         if self.sort:
             items.sort(key=lambda o: str(o[1]).upper())
-        if zero and not self.zero is None and not self.multiple:
+        if zero and self.zero is not None and not self.multiple:
             items.insert(0, ('', self.zero))
         return items
 
@@ -823,7 +824,7 @@ class IS_INT_IN_RANGE(Validator):
 
 def str2dec(number):
     s = str(number)
-    if not '.' in s:
+    if '.' not in s:
         s += '.00'
     else:
         s += '0' * (2 - len(s.split('.')[1]))
@@ -1213,7 +1214,7 @@ class IS_EMAIL(Validator):
                 domain_encoded = to_unicode(domain).encode('idna').decode('ascii')
                 match_domain = self.domain_regex.match(domain_encoded)
 
-            match = (match_body != None) and (match_domain != None)
+            match = (match_body is not None) and (match_domain is not None)
         except (TypeError, UnicodeError):
             # Value may not be a string where we can look for matches.
             # Example: we're calling ANY_OF formatter and IS_EMAIL is asked to validate a date.
@@ -1247,7 +1248,7 @@ class IS_LIST_OF_EMAILS(object):
         f = IS_EMAIL()
         for email in self.split_emails.findall(value):
             error = f(email)[1]
-            if error and not email in bad_emails:
+            if error and email not in bad_emails:
                 bad_emails.append(email)
         if not bad_emails:
             return (value, None)
@@ -1461,9 +1462,9 @@ def unicode_to_ascii_authority(authority):
         if label:
             asciiLabels.append(to_native(encodings.idna.ToASCII(label)))
         else:
-             # encodings.idna.ToASCII does not accept an empty string, but
-             # it is necessary for us to allow for empty labels so that we
-             # don't modify the URL
+            # encodings.idna.ToASCII does not accept an empty string, but
+            # it is necessary for us to allow for empty labels so that we
+            # don't modify the URL
             asciiLabels.append('')
     # RFC 3490, Section 4, Step 5
     return str(reduce(lambda x, y: x + unichr(0x002E) + y, asciiLabels))
@@ -1527,13 +1528,17 @@ def unicode_to_ascii_url(url, prepend_scheme):
     if prepended:
         scheme = ''
 
-    unparsed = urlparse.urlunparse((scheme, unicode_to_ascii_authority(authority), escape_unicode(path), '', escape_unicode(query), str(fragment)))
+    unparsed = urlparse.urlunparse((scheme,
+                                    unicode_to_ascii_authority(authority),
+                                    escape_unicode(path),
+                                    '',
+                                    escape_unicode(query),
+                                    str(fragment)))
     if unparsed.startswith('//'):
-        unparsed = unparsed[2:] # Remove the // urlunparse puts in the beginning
+        unparsed = unparsed[2:]  # Remove the // urlunparse puts in the beginning
     return unparsed
 
 
-
 class IS_GENERIC_URL(Validator):
     """
     Rejects a URL string if any of the following is true:
@@ -2622,7 +2627,7 @@ class ANY_OF(Validator):
     def __call__(self, value):
         for validator in self.subs:
             value, error = validator(value)
-            if error == None:
+            if error is None:
                 break
         return value, error
 
@@ -2762,7 +2767,7 @@ class LazyCrypt(object):
         else:
             digest_alg, key = self.crypt.digest_alg, ''
         if self.crypt.salt:
-            if self.crypt.salt == True:
+            if self.crypt.salt is True:
                 salt = str(web2py_uuid()).replace('-', '')[-16:]
             else:
                 salt = self.crypt.salt
@@ -2847,7 +2852,7 @@ class CRYPT(object):
     Supports standard algorithms
 
         >>> for alg in ('md5','sha1','sha256','sha384','sha512'):
-        ...     print str(CRYPT(digest_alg=alg,salt=True)('test')[0])
+        ...     print(str(CRYPT(digest_alg=alg,salt=True)('test')[0]))
         md5$...$...
         sha1$...$...
         sha256$...$...
@@ -2859,13 +2864,13 @@ class CRYPT(object):
     Supports for pbkdf2
 
         >>> alg = 'pbkdf2(1000,20,sha512)'
-        >>> print str(CRYPT(digest_alg=alg,salt=True)('test')[0])
+        >>> print(str(CRYPT(digest_alg=alg,salt=True)('test')[0]))
         pbkdf2(1000,20,sha512)$...$...
 
     An optional hmac_key can be specified and it is used as salt prefix
 
         >>> a = str(CRYPT(digest_alg='md5',key='mykey',salt=True)('test')[0])
-        >>> print a
+        >>> print(a)
         md5$...$...
 
     Even if the algorithm changes the hash can still be validated

gluon/widget.py

@@ -140,7 +140,7 @@ class web2pyDialog(object):
         else:
             import tkinter
             from tkinter import messagebox
-        
+
 
         bg_color = 'white'
         root.withdraw()
@@ -463,7 +463,7 @@ class web2pyDialog(object):
             import tkMessageBox as messagebox
         else:
             from tkinter import messagebox
-        
+
         messagebox.showerror('web2py start server', message)
 
     def start(self):
@@ -1076,7 +1076,10 @@ def start_schedulers(options):
         return
 
     # Work around OS X problem: http://bugs.python.org/issue9405
-    import urllib
+    if PY2:
+        import urllib
+    else:
+        import urllib.request as urllib
     urllib.getproxies()
 
     for app in apps:

scripts/setup-web2py-nginx-uwsgi-ubuntu.sh

@@ -222,7 +222,7 @@ echo <<EOF
 you can stop uwsgi and nginx with
 
   sudo /etc/init.d/nginx stop
-  sudo systemctl start emperor.uwsgi.service
+  sudo systemctl stop emperor.uwsgi.service
  
 and start it with