R-2.15.2

Fixes 6 issues

.travis.yml

@@ -4,11 +4,18 @@ sudo: required
 
 cache: pip
 
+dist: "trusty"
+
 python:
   - '2.7'
-  - 'pypy'
   - '3.5'
   - '3.6'
+  - 'pypy-5.3.1'
+  - 'pypy3.5-5.7.1-beta'
+
+matrix:
+  allow_failures:
+    - python: 'pypy3.5-5.7.1-beta'
 
 install:
   - pip install -e .
@@ -33,3 +40,6 @@ notifications:
 
 addons:
   postgresql: "9.4"
+  apt:
+    packages:
+      - postgresql-9.4-postgis-2.3

CHANGELOG

@@ -1,7 +1,11 @@
-## 2.16.0b1
+## 2.15.0b1
+- dropped support for python 2.6
+- dropped web shell
 - experimental python 3 support
 - experimental authapi for service login
+- allow ajax file uploads
 - more tests
+- more pep8 compliance
 - d3.js model visulization
 - improved scheduler
 - is_email support for internationalized Domain Names
@@ -20,6 +24,13 @@
 - Updated fpdf to latest version
 - JWT support
 - import fabfile for remote deployment
+- scheduler new feature: you can now specify intervals with cron
+- gluon/* removed from sys.path. Applications relying on statements like e.g.
+  "from storage import Storage"
+  will need to be rewritten with
+  "from gluon.storage import Storage"
+- tests can only be run with the usual web2py.py --run_system_tests OR with
+  python -m unittest -v gluon.tests on the root dir
 - jQuery 3.2.1
 - PyDAL 17.07 including:
   allow jsonb support for postgres
@@ -30,21 +41,8 @@
   improved Teradata support
   improved mongodb support
   overall refactoring
-  experimental support for Google Cloud SQL v2 (TODO)
+  experimental support for Google Cloud SQL v2
-
+  new pymysql driver
-## 2.15.x
-- web2py does not support python 2.6 anymore
-- py3.5 syntax compatible (see #1353 for details)
-- dropped web shell from admin
-- scheduler new feature: you can now specify intervals with cron
-- gluon/* removed from sys.path. Applications relying on statements like e.g.
-  "from storage import Storage"
-  will need to be rewritten with
-  "from gluon.storage import Storage"
-- tests can only be run with the usual web2py.py --run_system_tests OR with
-  python -m unittest -v gluon.tests on the root dir
-- updated pymysql driver
-
 
 ## 2.14.6
 

Makefile

@@ -32,7 +32,7 @@ update:
 	echo "remember that pymysql was tweaked"
 src:
 	### Use semantic versioning
-	echo 'Version 2.14.6-stable+timestamp.'`date +%Y.%m.%d.%H.%M.%S` > VERSION
+	echo 'Version 2.15.2-stable+timestamp.'`date +%Y.%m.%d.%H.%M.%S` > VERSION
 	### rm -f all junk files
 	make clean
 	### clean up baisc apps
@@ -54,7 +54,7 @@ src:
 	### build web2py_src.zip
 	echo '' > NEWINSTALL
 	mv web2py_src.zip web2py_src_old.zip | echo 'no old'
-	cd ..; zip -r web2py/web2py_src.zip web2py/web2py.py web2py/anyserver.py web2py/gluon/* web2py/extras/* web2py/handlers/* web2py/examples/* web2py/README.markdown  web2py/LICENSE web2py/CHANGELOG web2py/NEWINSTALL web2py/VERSION web2py/MANIFEST.in web2py/scripts/*.sh web2py/scripts/*.py web2py/applications/admin web2py/applications/examples/ web2py/applications/welcome web2py/applications/__init__.py web2py/site-packages/__init__.py web2py/gluon/tests/*.sh web2py/gluon/tests/*.py
+	cd ..; zip -r web2py/web2py_src.zip web2py/web2py.py web2py/anyserver.py web2py/fabfile.py web2py/gluon/* web2py/extras/* web2py/handlers/* web2py/examples/* web2py/README.markdown  web2py/LICENSE web2py/CHANGELOG web2py/NEWINSTALL web2py/VERSION web2py/MANIFEST.in web2py/scripts/*.sh web2py/scripts/*.py web2py/applications/admin web2py/applications/examples/ web2py/applications/welcome web2py/applications/__init__.py web2py/site-packages/__init__.py web2py/gluon/tests/*.sh web2py/gluon/tests/*.py
 
 mdp:
 	make src

VERSION

@@ -1 +1 @@
-Version 2.14.6-stable+timestamp.2016.05.09.19.18.48
+Version 2.15.2-stable+timestamp.2017.07.19.01.21.31

applications/admin/controllers/appadmin.py

@@ -690,6 +690,6 @@ def d3_graph_model():
         nodes.append(dict(name=tablename, type="table", fields = fields))
 
     # d3 v4 allows individual modules to be specified.  The complete d3 library is included below.
-    response.files.append(URL('static','js/d3.min.js'))
+    response.files.append(URL('admin','static','js/d3.min.js'))
-    response.files.append(URL('static','js/d3_graph.js'))
+    response.files.append(URL('admin','static','js/d3_graph.js'))
-    return dict(databases=databases, nodes=nodes, links=links)
+    return dict(databases=databases, nodes=nodes, links=links)

applications/admin/views/appadmin.html

@@ -239,7 +239,7 @@
     {{=T("No databases in this application")}}
   {{else:}}    
     <div id="vis"></div>
-      <link rel="stylesheet" href="{{=URL('static','css/d3_graph.css')}}"/>
+      <link rel="stylesheet" href="{{=URL('admin','static','css/d3_graph.css')}}"/>
       <script>
         // Define the d3 input data
         {{from gluon.serializers import json }}

applications/admin/views/default/site.html

@@ -46,7 +46,7 @@
       </td>
       <td>
         <div class="btn-group">
-          <a class="btn dropdown-toggle" data-toggle="dropdown" href="#">
+          <a class="btn dropdown-toggle" data-toggle="dropdown">
         {{=T('Manage')}}
         <span class="caret"></span>
           </a>

applications/examples/views/appadmin.html

@@ -239,7 +239,7 @@
     {{=T("No databases in this application")}}
   {{else:}}    
     <div id="vis"></div>
-      <link rel="stylesheet" href="{{=URL('static','css/d3_graph.css')}}"/>
+      <link rel="stylesheet" href="{{=URL('admin','static','css/d3_graph.css')}}"/>
       <script>
         // Define the d3 input data
         {{from gluon.serializers import json }}

applications/welcome/routes.example.py

@@ -20,8 +20,8 @@
 # YOU CAN COPY THIS FILE TO ANY APPLICATION'S ROOT DIRECTORY WITHOUT CHANGES!
 # ----------------------------------------------------------------------------------------------------------------------
 
-from fileutils import abspath
+from gluon.fileutils import abspath
-from languages import read_possible_languages
+from gluon.languages import read_possible_languages
 
 possible_languages = read_possible_languages(abspath('applications', app))
 # ----------------------------------------------------------------------------------------------------------------------

applications/welcome/static/js/web2py.js

@@ -12,6 +12,8 @@
         $.error('web2py.js has already been loaded!');
     }
 
+    var FORMDATA_IS_SUPPORTED = typeof(FormData) !== 'undefined';
+
     String.prototype.reverse = function () {
         return this.split('').reverse().join('');
     };
@@ -320,7 +322,15 @@
                 form.submit(function (e) {
                     web2py.disableElement(form.find(web2py.formInputClickSelector));
                     web2py.hide_flash();
-                    web2py.ajax_page('post', url, form.serialize(), target, form);
+
+                    var formData;
+                    if (FORMDATA_IS_SUPPORTED) {
+                        formData = new FormData(form[0]); // Allows file uploads.
+                    } else {
+                        formData = form.serialize(); // Fallback for older browsers.
+                    }
+                    web2py.ajax_page('post', url, formData, target, form);
+
                     e.preventDefault();
                 });
                 form.on('click', web2py.formInputClickSelector, function (e) {
@@ -339,11 +349,18 @@
             if (web2py.isUndefined(element)) element = $(document);
             /* if target is not there, fill it with something that there isn't in the page*/
             if (web2py.isUndefined(target) || target === '') target = 'w2p_none';
+
+            /* processData and contentType must be set to false when passing a FormData
+               object to jQuery.ajax. */
+            var isFormData = Object.prototype.toString.call(data) === '[object FormData]';
+            var contentType = isFormData ? false : 'application/x-www-form-urlencoded; charset=UTF-8';
             if (web2py.fire(element, 'ajax:before', null, target)) { /*test a usecase, should stop here if returns false */
                 $.ajax({
                     'type': method,
                     'url': action,
                     'data': data,
+                    'processData': !isFormData,
+                    'contentType': contentType,
                     'beforeSend': function (xhr, settings) {
                         xhr.setRequestHeader('web2py-component-location', document.location);
                         xhr.setRequestHeader('web2py-component-element', target);
@@ -699,8 +716,9 @@
             });
         },
         /* Disables form elements:
+        - Does not disable elements with 'data-w2p_disable' attribute
         - Caches element value in 'w2p_enable_with' data store
-        - Replaces element text with value of 'data-disable-with' attribute
+        - Replaces element text with value of 'data-w2p_disable_with' attribute
         - Sets disabled property to true
         */
         disableFormElements: function (form) {
@@ -712,13 +730,15 @@
                 if (!web2py.isUndefined(disable)) {
                     return false;
                 }
-                if (web2py.isUndefined(disable_with)) {
+                if (!element.is(':file')) { // Altering file input values is not allowed.
-                    element.data('w2p_disable_with', element[method]());
+                    if (web2py.isUndefined(disable_with)) {
+                        element.data('w2p_disable_with', element[method]());
+                    }
+                    if (web2py.isUndefined(element.data('w2p_enable_with'))) {
+                        element.data('w2p_enable_with', element[method]());
+                    }
+                    element[method](element.data('w2p_disable_with'));
                 }
-                if (web2py.isUndefined(element.data('w2p_enable_with'))) {
-                    element.data('w2p_enable_with', element[method]());
-                }
-                element[method](element.data('w2p_disable_with'));
                 element.prop('disabled', true);
             });
         },

applications/welcome/views/appadmin.html

@@ -239,7 +239,7 @@
     {{=T("No databases in this application")}}
   {{else:}}    
     <div id="vis"></div>
-      <link rel="stylesheet" href="{{=URL('static','css/d3_graph.css')}}"/>
+      <link rel="stylesheet" href="{{=URL('admin','static','css/d3_graph.css')}}"/>
       <script>
         // Define the d3 input data
         {{from gluon.serializers import json }}

gluon/admin.py

@@ -54,7 +54,8 @@ def app_pack(app, request, raise_ex=False, filenames=None):
 
     """
     try:
-        if filenames is None: app_cleanup(app, request)
+        if filenames is None:
+            app_cleanup(app, request)
         filename = apath('../deposit/web2py.app.%s.w2p' % app, request)
         w2p_pack(filename, apath(app, request), filenames=filenames)
         return filename
@@ -104,7 +105,8 @@ def app_cleanup(app, request):
     if os.path.exists(path):
         for f in os.listdir(path):
             try:
-                if f[:1] != '.': os.unlink(os.path.join(path, f))
+                if f[:1] != '.':
+                    os.unlink(os.path.join(path, f))
             except IOError:
                 r = False
 
@@ -113,7 +115,8 @@ def app_cleanup(app, request):
     if os.path.exists(path):
         for f in os.listdir(path):
             try:
-                if f[:1] != '.': recursive_unlink(os.path.join(path, f))
+                if f[:1] != '.':
+                    recursive_unlink(os.path.join(path, f))
             except (OSError, IOError):
                 r = False
 
@@ -123,7 +126,8 @@ def app_cleanup(app, request):
         CacheOnDisk(folder=path).clear()
         for f in os.listdir(path):
             try:
-                if f[:1] != '.': recursive_unlink(os.path.join(path, f))
+                if f[:1] != '.':
+                    recursive_unlink(os.path.join(path, f))
             except (OSError, IOError):
                 r = False
     return r
@@ -175,10 +179,9 @@ def app_create(app, request, force=False, key=None, info=False):
             return False
     try:
         w2p_unpack('welcome.w2p', path)
-        for subfolder in [
+        for subfolder in ['models', 'views', 'controllers', 'databases',
-            'models', 'views', 'controllers', 'databases',
+                          'modules', 'cron', 'errors', 'sessions', 'cache',
-            'modules', 'cron', 'errors', 'sessions', 'cache',
+                          'languages', 'static', 'private', 'uploads']:
-            'languages', 'static', 'private', 'uploads']:
             subpath = os.path.join(path, subfolder)
             if not os.path.exists(subpath):
                 os.mkdir(subpath)
@@ -368,7 +371,7 @@ def unzip(filename, dir, subfolder=''):
     for name in sorted(zf.namelist()):
         if not name.startswith(subfolder):
             continue
-        #print name[n:]
+        # print name[n:]
         if name.endswith('/'):
             folder = os.path.join(dir, name[n:])
             if not os.path.exists(folder):
@@ -435,6 +438,7 @@ def add_path_first(path):
     if not global_settings.web2py_runtime_gae:
         site.addsitedir(path)
 
+
 def try_mkdir(path):
     if not os.path.exists(path):
         try:
@@ -444,11 +448,12 @@ def try_mkdir(path):
             else:
                 os.mkdir(path)
         except OSError as e:
-            if e.strerror == 'File exists': # In case of race condition.
+            if e.strerror == 'File exists':  # In case of race condition.
                 pass
             else:
                 raise e
 
+
 def create_missing_folders():
     if not global_settings.web2py_runtime_gae:
         for path in ('applications', 'deposit', 'site-packages', 'logs'):

gluon/authapi.py

@@ -115,7 +115,7 @@ class AuthAPI(object):
             if auth.last_visit and auth.last_visit + delta > now:
                 self.user = auth.user
                 # this is a trick to speed up sessions to avoid many writes
-                if (now - auth.last_visit).seconds > (auth.expiration / 10):
+                if (now - auth.last_visit).seconds > (auth.expiration // 10):
                     auth.last_visit = now
             else:
                 self.user = None

gluon/compileapp.py

@@ -41,11 +41,12 @@ import imp
 import logging
 import types
 from functools import reduce
-logger = logging.getLogger("web2py")
 from gluon import rewrite
 from gluon.custom_import import custom_import_install
 import py_compile
 
+logger = logging.getLogger("web2py")
+
 is_pypy = settings.global_settings.is_pypy
 is_gae = settings.global_settings.web2py_runtime_gae
 is_jython = settings.global_settings.is_jython
@@ -111,7 +112,7 @@ class mybuiltin(object):
     NOTE could simple use a dict and populate it,
     NOTE not sure if this changes things though if monkey patching import.....
     """
-    #__builtins__
+    # __builtins__
     def __getitem__(self, key):
         try:
             return getattr(builtin, key)
@@ -185,7 +186,7 @@ def LOAD(c=None, f='index', args=None, vars=None,
         else:
             statement = "$.web2py.component('%s','%s');" % (url, target)
         attr['_data-w2p_remote'] = url
-        if not target is None:
+        if target is not None:
             return DIV(content, **attr)
 
     else:
@@ -211,7 +212,8 @@ def LOAD(c=None, f='index', args=None, vars=None,
             request.env.path_info
         other_request.cid = target
         other_request.env.http_web2py_component_element = target
-        other_request.restful = types.MethodType(request.restful.__func__, other_request) # A bit nasty but needed to use LOAD on action decorates with @request.restful()
+        other_request.restful = types.MethodType(request.restful.__func__, other_request)
+        # A bit nasty but needed to use LOAD on action decorates with @request.restful()
         other_response.view = '%s/%s.%s' % (c, f, other_request.extension)
 
         other_environment = copy.copy(current.globalenv)  # NASTY
@@ -405,7 +407,7 @@ def build_environment(request, response, session, store_current=True):
     """
     Build the environment dictionary into which web2py files are executed.
     """
-    #h,v = html,validators
+    # h,v = html,validators
     environment = dict(_base_environment_)
 
     if not request.env:
@@ -418,7 +420,7 @@ def build_environment(request, response, session, store_current=True):
         r'^%s/%s/\w+\.py$' % (request.controller, request.function)
         ]
 
-    t = environment['T'] = translator(os.path.join(request.folder,'languages'),
+    t = environment['T'] = translator(os.path.join(request.folder, 'languages'),
                                       request.env.http_accept_language)
     c = environment['cache'] = Cache(request)
 
@@ -506,10 +508,12 @@ def compile_models(folder):
         save_pyc(filename)
         os.unlink(filename)
 
+
 def find_exposed_functions(data):
-    data = regex_longcomments.sub('',data)
+    data = regex_longcomments.sub('', data)
     return regex_expose.findall(data)
 
+
 def compile_controllers(folder):
     """
     Compiles all the controllers in the application specified by `folder`
@@ -524,16 +528,19 @@ def compile_controllers(folder):
             command = data + "\nresponse._vars=response._caller(%s)\n" % \
                 function
             filename = pjoin(folder, 'compiled',
-                             'controllers.%s.%s.py' % (fname[:-3],function))
+                             'controllers.%s.%s.py' % (fname[:-3], function))
             write_file(filename, command)
             save_pyc(filename)
             os.unlink(filename)
 
+
 def model_cmp(a, b, sep='.'):
     return cmp(a.count(sep), b.count(sep)) or cmp(a, b)
 
+
 def model_cmp_sep(a, b, sep=os.path.sep):
-    return model_cmp(a,b,sep)
+    return model_cmp(a, b, sep)
+
 
 def run_models_in(environment):
     """
@@ -544,7 +551,7 @@ def run_models_in(environment):
     request = current.request
     folder = request.folder
     c = request.controller
-    #f = environment['request'].function
+    # f = environment['request'].function
     response = current.response
 
     path = pjoin(folder, 'models')
@@ -557,9 +564,11 @@ def run_models_in(environment):
             models = sorted(listdir(path, '^\w+\.py$', 0, sort=False), model_cmp_sep)
     else:
         if compiled:
-            models = sorted(listdir(cpath, '^models[_.][\w.]+\.pyc$', 0), key=lambda f: '{0:03d}'.format(f.count('.')) + f)
+            models = sorted(listdir(cpath, '^models[_.][\w.]+\.pyc$', 0),
+                            key=lambda f: '{0:03d}'.format(f.count('.')) + f)
         else:
-            models = sorted(listdir(path, '^\w+\.py$', 0, sort=False), key=lambda f: '{0:03d}'.format(f.count(os.path.sep)) + f)
+            models = sorted(listdir(path, '^\w+\.py$', 0, sort=False),
+                            key=lambda f: '{0:03d}'.format(f.count(os.path.sep)) + f)
 
     models_to_run = None
     for model in models:
@@ -570,10 +579,10 @@ def run_models_in(environment):
         if models_to_run:
             if compiled:
                 n = len(cpath)+8
-                fname = model[n:-4].replace('.','/')+'.py'
+                fname = model[n:-4].replace('.', '/')+'.py'
             else:
                 n = len(path)+1
-                fname = model[n:].replace(os.path.sep,'/')
+                fname = model[n:].replace(os.path.sep, '/')
             if not regex.search(fname) and c != 'appadmin':
                 continue
             elif compiled:
@@ -583,6 +592,7 @@ def run_models_in(environment):
             ccode = getcfs(model, model, f)
             restricted(ccode, environment, layer=model)
 
+
 def run_controller_in(controller, function, environment):
     """
     Runs the controller.function() (for the app specified by
@@ -596,13 +606,13 @@ def run_controller_in(controller, function, environment):
     badc = 'invalid controller (%s/%s)' % (controller, function)
     badf = 'invalid function (%s/%s)' % (controller, function)
     if os.path.exists(cpath):
-        filename = pjoin(cpath, 'controllers.%s.%s.pyc'
+        filename = pjoin(cpath, 'controllers.%s.%s.pyc' % (controller, function))
-                         % (controller, function))
+        try:
-        if not os.path.exists(filename):
+            ccode = getcfs(filename, filename, lambda: read_pyc(filename))
+        except IOError:
             raise HTTP(404,
                        rewrite.THREAD_LOCAL.routes.error_message % badf,
                        web2py_error=badf)
-        ccode = getcfs(filename, filename, lambda: read_pyc(filename))
     elif function == '_TEST':
         # TESTING: adjust the path to include site packages
         from gluon.settings import global_settings
@@ -623,15 +633,15 @@ def run_controller_in(controller, function, environment):
         code += TEST_CODE
         ccode = compile2(code, filename)
     else:
-        filename = pjoin(folder, 'controllers/%s.py'
+        filename = pjoin(folder, 'controllers/%s.py' % controller)
-                                 % controller)
+        try:
-        if not os.path.exists(filename):
+            code = getcfs(filename, filename, lambda: read_file(filename))
+        except IOError:
             raise HTTP(404,
                        rewrite.THREAD_LOCAL.routes.error_message % badc,
                        web2py_error=badc)
-        code = getcfs(filename, filename, lambda: read_file(filename))
         exposed = find_exposed_functions(code)
-        if not function in exposed:
+        if function not in exposed:
             raise HTTP(404,
                        rewrite.THREAD_LOCAL.routes.error_message % badf,
                        web2py_error=badf)
@@ -678,7 +688,7 @@ def run_view_in(environment):
         layer = 'file stream'
     else:
         filename = pjoin(folder, 'views', view)
-        if os.path.exists(cpath): # compiled views
+        if os.path.exists(cpath):  # compiled views
             x = view.replace('/', '.')
             files = ['views.%s.pyc' % x]
             is_compiled = os.path.exists(pjoin(cpath, files[0]))
@@ -705,16 +715,19 @@ def run_view_in(environment):
             raise HTTP(404,
                        rewrite.THREAD_LOCAL.routes.error_message % badv,
                        web2py_error=badv)
-        layer = filename
-        # Compile the template
-        ccode = parse_template(view,
-                               pjoin(folder, 'views'),
-                               context=environment)
 
+        # if the view is not compiled
+        if not layer:
+            # Compile the template
+            ccode = parse_template(view,
+                                   pjoin(folder, 'views'),
+                                   context=environment)
+        layer = filename
     restricted(ccode, environment, layer=layer)
     # parse_template saves everything in response body
     return environment['response'].body.getvalue()
 
+
 def remove_compiled_application(folder):
     """
     Deletes the folder `compiled` containing the compiled application.

gluon/contenttype.py

@@ -330,7 +330,7 @@ CONTENT_TYPE = {
     '.lha': 'application/x-lha',
     '.lhs': 'text/x-literate-haskell',
     '.lhz': 'application/x-lhz',
-    '.load' : 'text/html',
+    '.load': 'text/html',
     '.log': 'text/x-log',
     '.lrz': 'application/x-lrzip',
     '.ltx': 'text/x-tex',
@@ -823,7 +823,7 @@ CONTENT_TYPE = {
     '.xsd': 'application/xml',
     '.xsl': 'application/xslt+xml',
     '.xslfo': 'text/x-xslfo',
-    '.xslm' : 'application/vnd.ms-excel.sheet.macroEnabled.12',
+    '.xslm': 'application/vnd.ms-excel.sheet.macroEnabled.12',
     '.xslt': 'application/xslt+xml',
     '.xspf': 'application/xspf+xml',
     '.xul': 'application/vnd.mozilla.xul+xml',
@@ -843,7 +843,7 @@ def contenttype(filename, default='text/plain'):
     """
     Returns the Content-Type string matching extension of the given filename.
     """
-    filename=to_native(filename)
+    filename = to_native(filename)
     i = filename.rfind('.')
     if i >= 0:
         default = CONTENT_TYPE.get(filename[i:].lower(), default)

gluon/contrib/heroku.py

@@ -7,15 +7,13 @@ db = get_db()
 """
 import os
 from gluon import *
-from pydal.adapters import ADAPTERS, PostgreSQLAdapter
+from pydal.adapters import adapters, PostgrePsyco
-from pydal.helpers.classes import UseDatabaseStoredFile
+from pydal.helpers.classes import DatabaseStoredFile
 
-class HerokuPostgresAdapter(UseDatabaseStoredFile,PostgreSQLAdapter):
+@adapters.register_for('postgres')
-    drivers = ('psycopg2',)
+class HerokuPostgresAdapter(DatabaseStoredFile, PostgrePsyco):
     uploads_in_blob = True
 
-ADAPTERS['postgres'] = HerokuPostgresAdapter
-
 def get_db(name = None, pool_size=10):
     if not name:
         names = [n for n in os.environ.keys()

gluon/contrib/imageutils.py

@@ -45,7 +45,7 @@ class RESIZE(object):
                 background = Image.new('RGBA', (self.nx, self.ny), (255, 255, 255, 0))
                 background.paste(
                     img,
-                    ((self.nx - img.size[0]) / 2, (self.ny - img.size[1]) / 2))
+                    ((self.nx - img.size[0]) // 2, (self.ny - img.size[1]) // 2))
                 background.save(s, 'JPEG', quality=self.quality)
             else:
                 img.save(s, 'JPEG', quality=self.quality)

gluon/contrib/markmin/markmin2html.py

@@ -5,12 +5,19 @@
 # license MIT/BSD/GPL
 from __future__ import print_function
 import re
+import sys
 import urllib
-from gluon._compat import maketrans, urllib_quote, unicodeT, to_bytes, to_native, xrange
-from gluon.utils import local_html_escape as escape
-from ast import parse as ast_parse
 import ast
 
+PY2 = sys.version_info[0] == 2
+
+if PY2:
+    from urllib import quote as urllib_quote
+    from string import maketrans
+else:
+    from urllib.parse import quote as urllib_quote
+    maketrans = str.maketrans
+
 
 """
 TODO: next version should use MathJax
@@ -564,6 +571,29 @@ ttab_in = maketrans("'`:*~\\[]{}@$+-.#\n", '\x0b\x0c\x0e\x0f\x10\x11\x12\x13\x14
 ttab_out = maketrans('\x0b\x0c\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x05', "'`:*~\\[]{}@$+-.#\n")
 regex_quote = re.compile('(?P<name>\w+?)\s*\=\s*')
 
+def local_html_escape(data, quote=False):
+    """
+    Works with bytes.
+    Replace special characters "&", "<" and ">" to HTML-safe sequences.
+    If the optional flag quote is true (the default), the quotation mark
+    characters, both double quote (") and single quote (') characters are also
+    translated.
+    """
+    if PY2:
+        import cgi
+        data = cgi.escape(data, quote)
+        return data.replace("'", "&#x27;") if quote else data
+    else:
+        import html
+        if isinstance(data, str):
+            return html.escape(data, quote=quote)
+        data = data.replace(b"&", b"&amp;")  # Must be done first!                                                                                           
+        data = data.replace(b"<", b"&lt;")
+        data = data.replace(b">", b"&gt;")
+        if quote:
+            data = data.replace(b'"', b"&quot;")
+            data = data.replace(b'\'', b"&#x27;")
+        return data
 
 def make_dict(b):
     return '{%s}' % regex_quote.sub("'\g<name>':", b)
@@ -579,7 +609,7 @@ def safe_eval(node_or_string, env):
     _safe_names = {'None': None, 'True': True, 'False': False}
     _safe_names.update(env)
     if isinstance(node_or_string, basestring):
-        node_or_string = ast_parse(node_or_string, mode='eval')
+        node_or_string = ast.parse(node_or_string, mode='eval')
     if isinstance(node_or_string, ast.Expression):
         node_or_string = node_or_string.body
 
@@ -599,14 +629,14 @@ def safe_eval(node_or_string, env):
             if node.id in _safe_names:
                 return _safe_names[node.id]
         elif isinstance(node, ast.BinOp) and \
-                isinstance(node.op, (Add, Sub)) and \
+                isinstance(node.op, (ast.Add, ast.Sub)) and \
-                isinstance(node.right, Num) and \
+                isinstance(node.right, ast.Num) and \
                 isinstance(node.right.n, complex) and \
-                isinstance(node.left, Num) and \
+                isinstance(node.left, ast.Num) and \
                 isinstance(node.left.n, (int, long, float)):
             left = node.left.n
             right = node.right.n
-            if isinstance(node.op, Add):
+            if isinstance(node.op, ast.Add):
                 return left + right
             else:
                 return left - right
@@ -765,7 +795,7 @@ def render(text,
     '<table><tbody><tr class="first"><td>a</td><td>b</td></tr><tr class="even"><td>c</td><td>d</td></tr></tbody></table>'
 
     >>> render("----\\nhello world\\n----\\n")
-    '<blockquote>hello world</blockquote>'
+    '<blockquote><p>hello world</p></blockquote>'
 
     >>> render('[[myanchor]]')
     '<p><span class="anchor" id="markmin_myanchor"></span></p>'
@@ -946,7 +976,8 @@ def render(text,
     if protolinks == "default":
         protolinks = protolinks_simple
     pp = '\n' if pretty_print else ''
-    text = to_native(text)
+    text = text if text is None or isinstance(text, str) else text.decode('utf8', 'strict')
+
     if not (isinstance(text, str)):
         text = str(text or '')
     text = regex_backslash.sub(lambda m: m.group(1).translate(ttab_in), text)
@@ -994,7 +1025,7 @@ def render(text,
         return LINK
 
     text = regex_link.sub(mark_link, text)
-    text = escape(text)
+    text = local_html_escape(text)
 
     if protolinks:
         text = regex_proto.sub(lambda m: protolinks(*m.group('p', 'k')), text)
@@ -1035,7 +1066,7 @@ def render(text,
             if pend and mtag == '.':  # paragraph in a list:
                 out.append(etags.pop())
                 ltags.pop()
-            for i in xrange(lent - lev):
+            for i in range(lent - lev):
                 out.append('<' + tag + '>' + pp)
                 etags.append('</' + tag + '>' + pp)
                 lev += 1
@@ -1044,7 +1075,7 @@ def render(text,
         elif lent == lev:
             if tlev[-1] != tag:
                 # type of list is changed (ul<=>ol):
-                for i in xrange(ltags.count(lent)):
+                for i in range(ltags.count(lent)):
                     ltags.pop()
                     out.append(etags.pop())
                 tlev[-1] = tag
@@ -1209,7 +1240,7 @@ def render(text,
                 s = '<blockquote%s%s>%s</blockquote>%s' \
                     % (t_cls,
                        t_id,
-                       '\n'.join(strings[bq_begin:lineno]), pp)
+                       render('\n'.join(strings[bq_begin:lineno])), pp)
                 mtag = 'q'
         else:
             s = '<hr />'
@@ -1322,10 +1353,10 @@ def render(text,
         t, a, k, p, w = m.group('t', 'a', 'k', 'p', 'w')
         if not k:
             return m.group(0)
-        k = escape(k)
+        k = local_html_escape(k)
         t = t or ''
         style = 'width:%s' % w if w else ''
-        title = ' title="%s"' % escape(a).replace(META, DISABLED_META) if a else ''
+        title = ' title="%s"' % local_html_escape(a).replace(META, DISABLED_META) if a else ''
         p_begin = p_end = ''
         if p == 'center':
             p_begin = '<p style="text-align:center">'
@@ -1349,7 +1380,7 @@ def render(text,
                        autolinks, protolinks, class_prefix, id_prefix, pretty_print)
             return '<%(p)s controls="controls"%(title)s%(style)s><source src="%(k)s" />%(t)s</%(p)s>' \
                    % dict(p=p, title=title, style=style, k=k, t=t)
-        alt = ' alt="%s"' % escape(t).replace(META, DISABLED_META) if t else ''
+        alt = ' alt="%s"' % local_html_escape(t).replace(META, DISABLED_META) if t else ''
         return '%(begin)s<img src="%(k)s"%(alt)s%(title)s%(style)s />%(end)s' \
                % dict(begin=p_begin, k=k, alt=alt, title=title, style=style, end=p_end)
 
@@ -1358,12 +1389,12 @@ def render(text,
         if not k and not t:
             return m.group(0)
         t = t or ''
-        a = escape(a) if a else ''
+        a = local_html_escape(a) if a else ''
         if k:
             if '#' in k and ':' not in k.split('#')[0]:
                 # wikipage, not external url
                 k = k.replace('#', '#' + id_prefix)
-            k = escape(k)
+            k = local_html_escape(k)
             title = ' title="%s"' % a.replace(META, DISABLED_META) if a else ''
             target = ' target="_blank"' if p == 'popup' else ''
             t = render(t, {}, {}, 'br', URL, environment, latex, None,
@@ -1373,7 +1404,7 @@ def render(text,
         if t == 'NEWLINE' and not a:
             return '<br />' + pp
         return '<span class="anchor" id="%s">%s</span>' % (
-            escape(id_prefix + t),
+            local_html_escape(id_prefix + t),
             render(a, {}, {}, 'br', URL,
                    environment, latex, autolinks,
                    protolinks, class_prefix,
@@ -1399,7 +1430,7 @@ def render(text,
     def expand_meta(m):
         code, b, p, s = segments.pop(0)
         if code is None or m.group() == DISABLED_META:
-            return escape(s)
+            return local_html_escape(s)
         if b in extra:
             if code[:1] == '\n':
                 code = code[1:]
@@ -1411,7 +1442,7 @@ def render(text,
                 return str(extra[b](code))
         elif b == 'cite':
             return '[' + ','.join('<a href="#%s" class="%s">%s</a>' %
-                                  (id_prefix + d, b, d) for d in escape(code).split(',')) + ']'
+                                  (id_prefix + d, b, d) for d in local_html_escape(code).split(',')) + ']'
         elif b == 'latex':
             return LATEX % urllib_quote(code)
         elif b in html_colors:
@@ -1426,12 +1457,12 @@ def render(text,
                    % (fg, bg, render(code, {}, {}, 'br', URL, environment, latex,
                                      autolinks, protolinks, class_prefix, id_prefix, pretty_print))
         cls = ' class="%s%s"' % (class_prefix, b) if b and b != 'id' else ''
-        id = ' id="%s%s"' % (id_prefix, escape(p)) if p else ''
+        id = ' id="%s%s"' % (id_prefix, local_html_escape(p)) if p else ''
         beg = (code[:1] == '\n')
         end = [None, -1][code[-1:] == '\n']
         if beg and end:
-            return '<pre><code%s%s>%s</code></pre>%s' % (cls, id, escape(code[1:-1]), pp)
+            return '<pre><code%s%s>%s</code></pre>%s' % (cls, id, local_html_escape(code[1:-1]), pp)
-        return '<code%s%s>%s</code>' % (cls, id, escape(code[beg:end]))
+        return '<code%s%s>%s</code>' % (cls, id, local_html_escape(code[beg:end]))
 
     text = regex_expand_meta.sub(expand_meta, text)
 

gluon/contrib/minify/cssmin.py

@@ -10,8 +10,11 @@ Original author: Zachary Voase
 Modified for inclusion into web2py by: Ross Peoples <ross.peoples@gmail.com>
 """
 
+try:
+    from StringIO import StringIO
+except ImportError:
+    from io import StringIO
 
-from StringIO import StringIO  # The pure-Python StringIO supports unicode.
 import re
 
 

gluon/contrib/minify/minify.py

@@ -8,30 +8,40 @@ Created by: Ross Peoples <ross.peoples@gmail.com>
 Modified by: Massimo Di Pierro <massimo.dipierro@gmail.com>
 """
 
-import cssmin
+from . import cssmin
-import jsmin
+from . import jsmin
 import os
 import hashlib
 import re
+import sys
+PY2 = sys.version_info[0] == 2
 
+if PY2:
+    hashlib_md5 = hashlib.md5
+else:
+    hashlib_md5 = lambda s: hashlib.md5(bytes(s, 'utf8'))
+
+def open_py23(filename, mode):
+    if PY2:
+        f = open(filename, mode + 'b')
+    else:
+        f = open(filename, mode, encoding="utf8")
+    return f
 
 def read_binary_file(filename):
-    f = open(filename, 'rb')
+    f = open_py23(filename, 'r')
     data = f.read()
     f.close()
     return data
 
-
 def write_binary_file(filename, data):
-    f = open(filename, 'wb')
+    f = open_py23(filename, 'w')
     f.write(data)
     f.close()
 
-
 def fix_links(css, static_path):
     return re.sub(r'url\((["\'])\.\./', 'url(\\1' + static_path, css)
 
-
 def minify(files, path_info, folder, optimize_css, optimize_js,
            ignore_concat=[],
            ignore_minify=['/jquery.js', '/anytime.js']):
@@ -109,7 +119,7 @@ def minify(files, path_info, folder, optimize_css, optimize_js,
                     js.append(contents)
             else:
                 js.append(filename)
-    dest_key = hashlib.md5(repr(processed)).hexdigest()
+    dest_key = hashlib_md5(repr(processed)).hexdigest()
     if css and concat_css:
         css = '\n\n'.join(contents for contents in css)
         if not inline_css:

gluon/dal.py

@@ -14,6 +14,11 @@ from pydal import DAL as DAL
 from pydal import Field
 from pydal.objects import Row, Rows, Table, Query, Set, Expression
 from pydal import SQLCustomType, geoPoint, geoLine, geoPolygon
+from gluon.serializers import custom_json, xml
+from gluon.utils import web2py_uuid
+from gluon import sqlhtml
+from pydal.drivers import DRIVERS
+
 
 def _default_validators(db, field):
     """
@@ -78,14 +83,10 @@ def _default_validators(db, field):
     if (field.notnull or field.unique) and field_type not in excluded_fields:
         requires.insert(0, validators.IS_NOT_EMPTY())
     elif not field.notnull and not field.unique and requires:
-        requires[0] = validators.IS_EMPTY_OR(requires[0], null='' if field.type in ('string', 'text', 'password') else None)
+        requires[0] = \
+            validators.IS_EMPTY_OR(requires[0], null='' if field.type in ('string', 'text', 'password') else None)
     return requires
 
-from gluon.serializers import custom_json, xml
-from gluon.utils import web2py_uuid
-from gluon import sqlhtml
-
-
 DAL.serializers = {'json': custom_json, 'xml': xml}
 DAL.validators_method = _default_validators
 DAL.uuid = lambda x: web2py_uuid()
@@ -96,8 +97,7 @@ DAL.representers = {
 DAL.Field = Field
 DAL.Table = Table
 
-#: add web2py contrib drivers to pyDAL
+# add web2py contrib drivers to pyDAL
-from pydal.drivers import DRIVERS
 if not DRIVERS.get('pymysql'):
     try:
         from .contrib import pymysql

gluon/decoder.py

@@ -15,13 +15,13 @@ import codecs
 # None represents a potentially variable byte. "##" in the XML spec...
 autodetect_dict = {  # bytepattern     : ("name",
                                          (0x00, 0x00, 0xFE, 0xFF): ("ucs4_be"),
-    (0xFF, 0xFE, 0x00, 0x00): ("ucs4_le"),
+                                         (0xFF, 0xFE, 0x00, 0x00): ("ucs4_le"),
-    (0xFE, 0xFF, None, None): ("utf_16_be"),
+                                         (0xFE, 0xFF, None, None): ("utf_16_be"),
-    (0xFF, 0xFE, None, None): ("utf_16_le"),
+                                         (0xFF, 0xFE, None, None): ("utf_16_le"),
-    (0x00, 0x3C, 0x00, 0x3F): ("utf_16_be"),
+                                         (0x00, 0x3C, 0x00, 0x3F): ("utf_16_be"),
-    (0x3C, 0x00, 0x3F, 0x00): ("utf_16_le"),
+                                         (0x3C, 0x00, 0x3F, 0x00): ("utf_16_le"),
-    (0x3C, 0x3F, 0x78, 0x6D): ("utf_8"),
+                                         (0x3C, 0x3F, 0x78, 0x6D): ("utf_8"),
-    (0x4C, 0x6F, 0xA7, 0x94): ("EBCDIC")
+                                         (0x4C, 0x6F, 0xA7, 0x94): ("EBCDIC")
 }
 
 
@@ -36,10 +36,10 @@ def autoDetectXMLEncoding(buffer):
     # buffer at once but otherwise we'd have to decode a character at
     # a time looking for the quote character...that's a pain
 
-    encoding = "utf_8"    # according to the XML spec, this is the default
+    encoding = "utf_8"
-                          # this code successively tries to refine the default
+    # according to the XML spec, this is the default this code successively tries to refine the default
-                          # whenever it fails to refine, it falls back to
+    # whenever it fails to refine, it falls back to the last place encoding was set.
-                          # the last place encoding was set.
+
     if len(buffer) >= 4:
         bytes = (byte1, byte2, byte3, byte4) = tuple(map(ord, buffer[0:4]))
         enc_info = autodetect_dict.get(bytes, None)
@@ -51,8 +51,7 @@ def autoDetectXMLEncoding(buffer):
         enc_info = None
 
     if enc_info:
-        encoding = enc_info  # we've got a guess... these are
+        encoding = enc_info  # we've got a guess... these are the new defaults
-                             #the new defaults
 
         # try to find a more precise encoding using xml declaration
         secret_decoder_ring = codecs.lookup(encoding)[1]

gluon/fileutils.py

@@ -415,10 +415,10 @@ def fix_newlines(path):
 |\r|
 )''')
     for filename in listdir(path, '.*\.(py|html)$', drop=False):
-        rdata = read_file(filename, 'rb')
+        rdata = read_file(filename, 'r')
         wdata = regex.sub('\n', rdata)
         if wdata != rdata:
-            write_file(filename, wdata, 'wb')
+            write_file(filename, wdata, 'w')
 
 
 def copystream(

gluon/globals.py

@@ -13,7 +13,8 @@ Contains the classes for the global used variables:
 - Session
 
 """
-from gluon._compat import pickle, StringIO, copyreg, Cookie, urlparse, PY2, iteritems, to_unicode, to_native, unicodeT, long
+from gluon._compat import pickle, StringIO, copyreg, Cookie, urlparse, PY2, iteritems, to_unicode, to_native, \
+    unicodeT, long, hashlib_md5
 from gluon.storage import Storage, List
 from gluon.streamer import streamer, stream_file_or_304_or_206, DEFAULT_CHUNK_SIZE
 from gluon.contenttype import contenttype
@@ -30,7 +31,7 @@ from gluon.fileutils import copystream
 import hashlib
 from pydal.contrib import portalocker
 from pickle import Pickler, MARK, DICT, EMPTY_DICT
-#from types import DictionaryType
+# from types import DictionaryType
 import datetime
 import re
 import os
@@ -48,7 +49,7 @@ PAST = 'Sat, 1-Jan-1971 00:00:00'
 FUTURE = 'Tue, 1-Dec-2999 23:59:59'
 
 try:
-    #FIXME PY3
+    # FIXME PY3
     from gluon.contrib.minify import minify
     have_minify = True
 except ImportError:
@@ -79,6 +80,7 @@ template_mapping = {
     'js:inline': js_inline
 }
 
+
 # IMPORTANT:
 # this is required so that pickled dict(s) and class.__dict__
 # are sorted and web2py can detect without ambiguity when a session changes
@@ -95,6 +97,7 @@ else:
     SortingPickler.dispatch_table = copyreg.dispatch_table.copy()
     SortingPickler.dispatch_table[dict] = SortingPickler.save_dict
 
+
 def sorting_dumps(obj, protocol=None):
     file = StringIO()
     SortingPickler(file, protocol).dump(obj)
@@ -120,7 +123,7 @@ def copystream_progress(request, chunk_size=10 ** 5):
         dest = tempfile.NamedTemporaryFile()
     except NotImplementedError:  # and GAE this
         dest = tempfile.TemporaryFile()
-    if not 'X-Progress-ID' in request.get_vars:
+    if 'X-Progress-ID' not in request.get_vars:
         copystream(source, dest, size, chunk_size)
         return dest
     cache_key = 'X-Progress-ID:' + request.get_vars['X-Progress-ID']
@@ -198,7 +201,8 @@ class Request(Storage):
         """Takes the QUERY_STRING and unpacks it to get_vars
         """
         query_string = self.env.get('query_string', '')
-        dget = urlparse.parse_qs(query_string, keep_blank_values=1)  # Ref: https://docs.python.org/2/library/cgi.html#cgi.parse_qs
+        dget = urlparse.parse_qs(query_string, keep_blank_values=1)
+        # Ref: https://docs.python.org/2/library/cgi.html#cgi.parse_qs
         get_vars = self._get_vars = Storage(dget)
         for (key, value) in iteritems(get_vars):
             if isinstance(value, list) and len(value) == 1:
@@ -228,8 +232,7 @@ class Request(Storage):
             body.seek(0)
 
         # parse POST variables on POST, PUT, BOTH only in post_vars
-        if (body and not is_json
+        if body and not is_json and env.request_method in ('POST', 'PUT', 'DELETE', 'BOTH'):
-            and env.request_method in ('POST', 'PUT', 'DELETE', 'BOTH')):
             query_string = env.pop('QUERY_STRING', None)
             dpost = cgi.FieldStorage(fp=body, environ=env, keep_blank_values=1)
             try:
@@ -355,7 +358,7 @@ class Request(Storage):
             def f(_action=action, *a, **b):
                 request.is_restful = True
                 env = request.env
-                is_json = env.content_type=='application/json'
+                is_json = env.content_type == 'application/json'
                 method = env.request_method
                 if not ignore_extension and len(request.args) and '.' in request.args[-1]:
                     request.args[-1], _, request.extension = request.args[-1].rpartition('.')
@@ -451,13 +454,13 @@ class Response(Storage):
         for meta in iteritems((self.meta or {})):
             k, v = meta
             if isinstance(v, dict):
-                s += '<meta' + ''.join(' %s="%s"' % (xmlescape(key), to_native(xmlescape(v[key]))) for key in v) +' />\n'
+                s += '<meta' + ''.join(' %s="%s"' % (xmlescape(key),
+                                                     to_native(xmlescape(v[key]))) for key in v) + ' />\n'
             else:
                 s += '<meta name="%s" content="%s" />\n' % (k, to_native(xmlescape(v)))
         self.write(s, escape=False)
 
     def include_files(self, extensions=None):
-
         """
         Includes files (usually in the head).
         Can minify and cache local files
@@ -484,8 +487,7 @@ class Response(Storage):
 
         if have_minify and ((self.optimize_css and has_css) or (self.optimize_js and has_js)):
             # cache for 5 minutes by default
-            key = hashlib.md5(repr(files)).hexdigest()
+            key = hashlib_md5(repr(files)).hexdigest()
-
             cache = self.cache_includes or (current.cache.ram, 60 * 5)
 
             def call_minify(files=files):
@@ -523,6 +525,7 @@ class Response(Storage):
                 tmpl = template_mapping.get(f)
                 if tmpl:
                     s.append(tmpl % item[1])
+
         self.write(''.join(s), escape=False)
 
     def stream(self,
@@ -578,9 +581,9 @@ class Response(Storage):
         if hasattr(stream, 'name'):
             filename = stream.name
 
-        if filename and not 'content-type' in keys:
+        if filename and 'content-type' not in keys:
             headers['Content-Type'] = contenttype(filename)
-        if filename and not 'content-length' in keys:
+        if filename and 'content-length' not in keys:
             try:
                 headers['Content-Length'] = \
                     os.path.getsize(filename)
@@ -1022,7 +1025,7 @@ class Session(Storage):
         if self._forget:
             del rcookies[response.session_id_name]
             return
-        if self.get('httponly_cookies',True):
+        if self.get('httponly_cookies', True):
             scookies['HttpOnly'] = True
         if self._secure:
             scookies['secure'] = True
@@ -1193,7 +1196,7 @@ class Session(Storage):
             if (not response.session_id or
                 not response.session_filename or
                 self._forget
-                or self._unchanged(response)):
+                    or self._unchanged(response)):
                 # self.clear_session_cookies()
                 return False
             else:

gluon/highlight.py

@@ -182,8 +182,8 @@ class Highlighter(object):
         )),
         'PYTHONMultilineString': (python_tokenizer,
                                   (('ENDMULTILINESTRING',
-                                  re.compile(r'.*?("""|\'\'\')',
+                                    re.compile(r'.*?("""|\'\'\')',
-                                  re.DOTALL), 'color: darkred'), )),
+                                               re.DOTALL), 'color: darkred'), )),
         'HTML': (html_tokenizer, (
             ('GOTOPYTHON', re.compile(r'\{\{'), 'color: red'),
             ('COMMENT', re.compile(r'<!--[^>]*-->|<!>'),
@@ -209,7 +209,7 @@ class Highlighter(object):
         mode = self.mode
         while i < len(data):
             for (token, o_re, style) in Highlighter.all_styles[mode][1]:
-                if not token in self.suppress_tokens:
+                if token not in self.suppress_tokens:
                     match = o_re.match(data, i)
                     if match:
                         if style:
@@ -221,7 +221,7 @@ class Highlighter(object):
                             new_mode = \
                                 Highlighter.all_styles[mode][0](self,
                                                                 token, match, style)
-                        if not new_mode is None:
+                        if new_mode is not None:
                             mode = new_mode
                         i += max(1, len(match.group()))
                         break
@@ -241,9 +241,9 @@ class Highlighter(object):
             style = self.styles[token]
         if self.span_style != style:
             if style != 'Keep':
-                if not self.span_style is None:
+                if self.span_style is not None:
                     self.output.append('</span>')
-                if not style is None:
+                if style is not None:
                     self.output.append('<span style="%s">' % style)
                 self.span_style = style
 
@@ -260,7 +260,7 @@ def highlight(
 ):
     styles = styles or {}
     attributes = attributes or {}
-    if not 'CODE' in styles:
+    if 'CODE' not in styles:
         code_style = """
         font-size: 11px;
         font-family: Bitstream Vera Sans Mono,monospace;
@@ -272,7 +272,7 @@ def highlight(
         white-space: pre !important;\n"""
     else:
         code_style = styles['CODE']
-    if not 'LINENUMBERS' in styles:
+    if 'LINENUMBERS' not in styles:
         linenumbers_style = """
         font-size: 11px;
         font-family: Bitstream Vera Sans Mono,monospace;
@@ -283,7 +283,7 @@ def highlight(
         color: #A0A0A0;\n"""
     else:
         linenumbers_style = styles['LINENUMBERS']
-    if not 'LINEHIGHLIGHT' in styles:
+    if 'LINEHIGHLIGHT' not in styles:
         linehighlight_style = "background-color: #EBDDE2;"
     else:
         linehighlight_style = styles['LINEHIGHLIGHT']
@@ -333,8 +333,9 @@ def highlight(
                   == '_' and value])
     if fa:
         fa = ' ' + fa
-    return '<table%s><tr style="vertical-align:top;"><td style="min-width:40px; text-align: right;"><pre style="%s">%s</pre></td><td><pre style="%s">%s</pre></td></tr></table>'\
+    return '<table%s><tr style="vertical-align:top;">' \
-        % (fa, linenumbers_style, numbers, code_style, code)
+           '<td style="min-width:40px; text-align: right;"><pre style="%s">%s</pre></td>' \
+           '<td><pre style="%s">%s</pre></td></tr></table>' % (fa, linenumbers_style, numbers, code_style, code)
 
 
 if __name__ == '__main__':
@@ -342,5 +343,4 @@ if __name__ == '__main__':
     argfp = open(sys.argv[1])
     data = argfp.read()
     argfp.close()
-    print('<html><body>' + highlight(data, sys.argv[2])\
+    print('<html><body>' + highlight(data, sys.argv[2]) + '</body></html>')
-        + '</body></html>')

gluon/html.py

@@ -20,7 +20,8 @@ import urllib
 import base64
 from gluon import sanitizer, decoder
 import itertools
-from gluon._compat import reduce, pickle, copyreg, HTMLParser, name2codepoint, iteritems, unichr, unicodeT, urllib_quote, to_bytes, to_native, to_unicode, basestring, urlencode, implements_bool, text_type, long
+from gluon._compat import reduce, pickle, copyreg, HTMLParser, name2codepoint, iteritems, unichr, unicodeT, \
+    urllib_quote, to_bytes, to_native, to_unicode, basestring, urlencode, implements_bool, text_type, long
 from gluon.utils import local_html_escape
 import marshal
 
@@ -109,6 +110,7 @@ __all__ = [
 
 DEFAULT_PASSWORD_DISPLAY = '*' * 8
 
+
 def xmlescape(data, quote=True):
     """
     Returns an escaped string of the provided data
@@ -124,10 +126,9 @@ def xmlescape(data, quote=True):
 
     if not(isinstance(data, (text_type, bytes))):
         # i.e., integers
-        data=str(data)
+        data = str(data)
     data = to_bytes(data, 'utf8', 'xmlcharrefreplace')
 
-
     # ... and do the escaping
     data = local_html_escape(data, quote)
     return data
@@ -671,6 +672,7 @@ def XML_pickle(data):
     return XML_unpickle, (marshal.dumps(str(data)),)
 copyreg.pickle(XML, XML_pickle, XML_unpickle)
 
+
 @implements_bool
 class DIV(XmlComponent):
     """
@@ -1309,8 +1311,10 @@ class HTML(DIV):
     tag = b'html'
 
     strict = b'<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">\n'
-    transitional = b'<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">\n'
+    transitional = \
-    frameset = b'<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Frameset//EN" "http://www.w3.org/TR/html4/frameset.dtd">\n'
+        b'<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">\n'
+    frameset = \
+        b'<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Frameset//EN" "http://www.w3.org/TR/html4/frameset.dtd">\n'
     html5 = b'<!DOCTYPE HTML>\n'
 
     def xml(self):
@@ -1861,7 +1865,7 @@ class INPUT(DIV):
                 except:
                     import traceback
                     print(traceback.format_exc())
-                    msg = "Validation error, field:%s %s" % (name,validator)
+                    msg = "Validation error, field:%s %s" % (name, validator)
                     raise Exception(msg)
                 if errors is not None:
                     self.vars[name] = value
@@ -1911,7 +1915,7 @@ class INPUT(DIV):
         name = self.attributes.get('_name', None)
         if name and hasattr(self, 'errors') \
                 and self.errors.get(name, None) \
-                and self['hideerror'] != True:
+                and self['hideerror'] is not True:
             self['_class'] = (self['_class'] and self['_class'] + ' ' or '') + 'invalidinput'
             return DIV.xml(self) + DIV(
                 DIV(
@@ -1979,7 +1983,6 @@ class OPTGROUP(DIV):
 
 
 class SELECT(INPUT):
-
     """
     Examples:
 
@@ -2014,7 +2017,7 @@ class SELECT(INPUT):
         if value is not None:
             if not self['_multiple']:
                 for c in options:  # my patch
-                    if ((value is not None) and (str(c['_value']) == str(value))):
+                    if (value is not None) and (str(c['_value']) == str(value)):
                         c['_selected'] = 'selected'
                     else:
                         c['_selected'] = None
@@ -2024,7 +2027,7 @@ class SELECT(INPUT):
                 else:
                     values = [str(value)]
                 for c in options:  # my patch
-                    if ((value is not None) and (str(c['_value']) in values)):
+                    if (value is not None) and (str(c['_value']) in values):
                         c['_selected'] = 'selected'
                     else:
                         c['_selected'] = None
@@ -2390,7 +2393,6 @@ class FORM(DIV):
 
 
 class BEAUTIFY(DIV):
-
     """
     Turns any list, dictionary, etc into decent looking html.
 
@@ -2547,7 +2549,7 @@ class MENU(DIV):
                         li['_class'] = li['_class'] + ' ' + self['li_active']
                     else:
                         li['_class'] = self['li_active']
-                if len(item) <= 4 or item[4] == True:
+                if len(item) <= 4 or item[4] is True:
                     ul.append(li)
         return ul
 
@@ -2561,7 +2563,7 @@ class MENU(DIV):
                 # ex: ('', False, A('title', _href=URL(...), _title="title"))
                 # ex: (A('title', _href=URL(...), _title="title"), False, None)
                 custom_items.append(item)
-            elif len(item) <= 4 or item[4] == True:
+            elif len(item) <= 4 or item[4] is True:
                 select.append(OPTION(CAT(prefix, item[0]),
                                      _value=item[2], _selected=item[1]))
                 if len(item) > 3 and len(item[3]):
@@ -2703,7 +2705,8 @@ class web2pyHTMLParser(HTMLParser):
                 self.parent = self.parent.parent
             except:
                 raise RuntimeError("unable to balance tag %s" % tagname)
-            if parent_tagname[:len(tagname)] == tagname: break
+            if parent_tagname[:len(tagname)] == tagname:
+                break
 
 
 def markdown_serializer(text, tag=None, attr=None):

gluon/http.py

@@ -11,7 +11,7 @@ HTTP statuses helpers
 """
 
 import re
-from gluon._compat import iteritems
+from gluon._compat import iteritems, unicodeT
 
 __all__ = ['HTTP', 'redirect']
 
@@ -116,12 +116,14 @@ class HTTP(Exception):
         for k, v in iteritems(headers):
             if isinstance(v, list):
                 rheaders += [(k, str(item)) for item in v]
-            elif not v is None:
+            elif v is not None:
                 rheaders.append((k, str(v)))
         responder(status, rheaders)
         if env.get('request_method', '') == 'HEAD':
             return ['']
         elif isinstance(body, (str, bytes, bytearray)):
+            if isinstance(body, unicodeT):
+                body = body.encode('utf-8')
             return [body]
         elif hasattr(body, '__iter__'):
             return body
@@ -148,7 +150,7 @@ class HTTP(Exception):
             web2py_error=self.headers.get('web2py_error'))
 
     def __str__(self):
-        "stringify me"
+        """stringify me"""
         return self.message
 
 

gluon/import_all.py

@@ -78,13 +78,9 @@ alert_dependency = ['hashlib', 'uuid']
 # Now we remove the blacklisted modules if we are using the stated
 # python version.
 #
-# List of modules deprecated in Python 2.6 or 2.7 that are in the above set
+# List of modules deprecated in Python 2.7 that are in the above list
 py27_deprecated = ['mhlib', 'multifile', 'mimify', 'sets', 'MimeWriter']  # And ['optparse'] but we need it for now
 
-if python_version >= '2.6':
-    base_modules += ['json', 'multiprocessing']
-    base_modules = list(set(base_modules).difference(set(py26_deprecated)))
-
 if python_version >= '2.7':
     base_modules += ['argparse', 'json', 'multiprocessing']
     base_modules = list(set(base_modules).difference(set(py27_deprecated)))

gluon/main.py

@@ -11,7 +11,8 @@ The gluon wsgi application
 """
 from __future__ import print_function
 
-if False: import import_all # DO NOT REMOVE PART OF FREEZE PROCESS
+if False:
+    import import_all  # DO NOT REMOVE PART OF FREEZE PROCESS
 import gc
 
 import os
@@ -26,7 +27,7 @@ import random
 import string
 
 from gluon._compat import Cookie, urllib2
-#from thread import allocate_lock
+# from thread import allocate_lock
 
 from gluon.fileutils import abspath, write_file
 from gluon.settings import global_settings
@@ -67,14 +68,14 @@ import gluon.messageboxhandler
 logging.gluon = gluon
 # so we must restore it! Thanks ozancag
 import locale
-locale.setlocale(locale.LC_CTYPE, "C") # IMPORTANT, web2py requires locale "C"
+locale.setlocale(locale.LC_CTYPE, "C")  # IMPORTANT, web2py requires locale "C"
 
 exists = os.path.exists
 pjoin = os.path.join
 
 try:
     logging.config.fileConfig(abspath("logging.conf"))
-except: # fails on GAE or when logfile is missing
+except:  # fails on GAE or when logfile is missing
     logging.basicConfig()
 logger = logging.getLogger("web2py")
 
@@ -254,6 +255,7 @@ class LazyWSGI(object):
                 return [data]
             for item in middleware_apps:
                 app = item(app)
+
             def caller(app):
                 return app(self.environ, self.start_response)
             return lambda caller=caller, app=app: caller(app)
@@ -294,9 +296,9 @@ def wsgibase(environ, responder):
     response = Response()
     session = Session()
     env = request.env
-    #env.web2py_path = global_settings.applications_parent
+    # env.web2py_path = global_settings.applications_parent
     env.web2py_version = web2py_version
-    #env.update(global_settings)
+    # env.update(global_settings)
     static_file = False
     http_response = None
     try:
@@ -325,7 +327,6 @@ def wsgibase(environ, responder):
                             'Expires'] = 'Thu, 31 Dec 2037 23:59:59 GMT'
                     response.stream(static_file, request=request)
 
-
                 # ##################################################
                 # fill in request items
                 # ##################################################
@@ -356,17 +357,15 @@ def wsgibase(environ, responder):
                 cmd_opts = global_settings.cmd_options
 
                 request.update(
-                    client = client,
+                    client=client,
-                    folder = abspath('applications', app) + os.sep,
+                    folder=abspath('applications', app) + os.sep,
-                    ajax = x_req_with == 'xmlhttprequest',
+                    ajax=x_req_with == 'xmlhttprequest',
-                    cid = env.http_web2py_component_element,
+                    cid=env.http_web2py_component_element,
-                    is_local = (env.remote_addr in local_hosts and
+                    is_local=(env.remote_addr in local_hosts and client == env.remote_addr),
-                                client == env.remote_addr),
+                    is_shell=False,
-                    is_shell = False,
+                    is_scheduler=False,
-                    is_scheduler = False,
+                    is_https=env.wsgi_url_scheme in HTTPS_SCHEMES or
-                    is_https = env.wsgi_url_scheme in HTTPS_SCHEMES or \
+                             request.env.http_x_forwarded_proto in HTTPS_SCHEMES or env.https == 'on'
-                        request.env.http_x_forwarded_proto in HTTPS_SCHEMES \
-                        or env.https == 'on'
                     )
                 request.url = environ['PATH_INFO']
 
@@ -390,7 +389,7 @@ def wsgibase(environ, responder):
                                    % 'invalid request',
                                    web2py_error='invalid application')
                 elif not request.is_local and exists(disabled):
-                    five0three = os.path.join(request.folder,'static','503.html')
+                    five0three = os.path.join(request.folder, 'static', '503.html')
                     if os.path.exists(five0three):
                         raise HTTP(503, file(five0three, 'r').read())
                     else:
@@ -406,7 +405,7 @@ def wsgibase(environ, responder):
                 # get the GET and POST data
                 # ##################################################
 
-                #parse_get_post_vars(request, environ)
+                # parse_get_post_vars(request, environ)
 
                 # ##################################################
                 # expose wsgi hooks for convenience
@@ -625,7 +624,7 @@ def appfactory(wsgiapp=wsgibase,
                 raise BaseException("Can't create dir %s" % profiler_dir)
         filepath = pjoin(profiler_dir, 'wtest')
         try:
-            filehandle = open( filepath, 'w' )
+            filehandle = open(filepath, 'w')
             filehandle.close()
             os.unlink(filepath)
         except IOError:

gluon/rewrite.py

@@ -206,8 +206,7 @@ def url_out(request, environ, application, controller, function,
     if host is True or (host is None and (scheme or port is not None)):
         host = request.env.http_host
     if not scheme or scheme is True:
-        scheme = request.env.get('wsgi_url_scheme', 'http').lower() \
+        scheme = request.env.get('wsgi_url_scheme', 'http').lower() if request else 'http'
-            if request else 'http'
     if host:
         host_port = host if not port else host.split(':', 1)[0] + ':%s' % port
         url = '%s://%s%s' % (scheme, host_port, url)

gluon/sqlhtml.py

@@ -405,7 +405,7 @@ class RadioWidget(OptionsWidget):
         cols = attributes.get('cols', 1)
         totals = len(options)
         mods = totals % cols
-        rows = totals / cols
+        rows = totals // cols
         if mods:
             rows += 1
 
@@ -471,7 +471,7 @@ class CheckboxesWidget(OptionsWidget):
         cols = attributes.get('cols', 1)
         totals = len(options)
         mods = totals % cols
-        rows = totals / cols
+        rows = totals // cols
         if mods:
             rows += 1
 
@@ -684,9 +684,10 @@ class AutocompleteWidget(object):
             urlvars = request.vars
             urlvars[default_var] = 1
             self.url = URL(args=request.args, vars=urlvars)
-            self.callback()
+            self.run_callback = True
         else:
             self.url = request
+            self.run_callback = False
 
     def callback(self):
         if self.keyword in self.request.vars:
@@ -759,6 +760,8 @@ class AutocompleteWidget(object):
                 raise HTTP(200, '')
 
     def __call__(self, field, value, **attributes):
+        if self.run_callback:
+            self.callback()
         default = dict(
             _type='text',
             value=(value is not None and str(value)) or '',
@@ -1916,7 +1919,7 @@ class SQLFORM(FORM):
         if 'table_name' in attributes:
             del attributes['table_name']
 
-        return SQLFORM(DAL(None).define_table(table_name, *fields),
+        return SQLFORM(DAL(None).define_table(table_name, *[field.clone() for field in fields]),
                        **attributes)
 
     @staticmethod

gluon/tests/test_appadmin.py

@@ -17,6 +17,7 @@ from gluon import fileutils
 from gluon.dal import DAL, Field, Table
 from gluon.http import HTTP
 from gluon.fileutils import open_file
+from gluon.cache import CacheInRam
 
 DEFAULT_URI = os.getenv('DB', 'sqlite:memory')
 
@@ -104,6 +105,21 @@ class TestAppAdmin(unittest.TestCase):
         self._test_index()
         remove_compiled_application(appname_path)
 
+    def test_index_minify(self):
+        # test for gluon/contrib/minify
+        self.env['response'].optimize_css = 'concat|minify'
+        self.env['response'].optimize_js = 'concat|minify'
+        self.env['current'].cache = Storage({'ram':CacheInRam()})
+        appname_path = os.path.join(os.getcwd(), 'applications', 'welcome')
+        self._test_index()
+        file_l = os.listdir(os.path.join(appname_path, 'static', 'temp'))
+        file_l.sort()
+        self.assertTrue(len(file_l) == 2)
+        self.assertEqual(file_l[0][0:10], 'compressed')
+        self.assertEqual(file_l[1][0:10], 'compressed')
+        self.assertEqual(file_l[0][-3:], 'css')
+        self.assertEqual(file_l[1][-2:], 'js')
+
     def test_select(self):
         request = self.env['request']
         request.args = List(['db'])

gluon/tests/test_fileutils.py

@@ -1,10 +1,11 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
+import os
 import unittest
 import datetime
 
-from gluon.fileutils import parse_version
+from gluon.fileutils import parse_version, fix_newlines
 
 
 class TestFileUtils(unittest.TestCase):
@@ -22,3 +23,6 @@ class TestFileUtils(unittest.TestCase):
         # Semantic Beta
         rtn = parse_version('Version 2.14.1-beta+timestamp.2016.03.21.22.35.26')
         self.assertEqual(rtn, (2, 14, 1, 'beta', datetime.datetime(2016, 3, 21, 22, 35, 26)))
+    
+    def test_fix_newlines(self):
+        fix_newlines(os.path.dirname(os.path.abspath(__file__)))

gluon/tools.py

@@ -1127,7 +1127,6 @@ def addrow(form, a, b, c, style, _id, position=-1):
 
 
 class AuthJWT(object):
-
     """
     Experimental!
 
@@ -1537,7 +1536,8 @@ class Auth(AuthAPI):
     # ## these are messages that can be customized
     default_messages = dict(AuthAPI.default_messages,
                             access_denied='Insufficient privileges',
-                            bulk_invite_body='You have been invited to join %(site)s, click %(link)s to complete the process',
+                            bulk_invite_body='You have been invited to join %(site)s, click %(link)s to complete '
+                                             'the process',
                             bulk_invite_subject='Invitation to join %(site)s',
                             delete_label='Check to delete',
                             email_sent='Email sent',
@@ -1762,7 +1762,7 @@ class Auth(AuthAPI):
             if auth.last_visit and auth.last_visit + delta > now:
                 self.user = auth.user
                 # this is a trick to speed up sessions to avoid many writes
-                if (now - auth.last_visit).seconds > (auth.expiration / 10):
+                if (now - auth.last_visit).seconds > (auth.expiration // 10):
                     auth.last_visit = now
             else:
                 self.user = None
@@ -1840,14 +1840,15 @@ class Auth(AuthAPI):
         # ## these are messages that can be customized
         messages = self.messages = Messages(current.T)
         messages.update(Auth.default_messages)
-        messages.update(ajax_failed_authentication=DIV(H4('NOT AUTHORIZED'),
+        messages.update(ajax_failed_authentication=
-                                                       'Please ',
+                        DIV(H4('NOT AUTHORIZED'),
-                                                       A('login',
+                            'Please ',
-                                                         _href=self.settings.login_url +
+                            A('login',
-                                                         ('?_next=' + urllib_quote(current.request.env.http_web2py_component_location))
+                              _href=self.settings.login_url +
-                                                         if current.request.env.http_web2py_component_location else ''),
+                                    ('?_next=' + urllib_quote(current.request.env.http_web2py_component_location))
-                                                       ' to view this content.',
+                              if current.request.env.http_web2py_component_location else ''),
-                                                       _class='not-authorized alert alert-block'))
+                            ' to view this content.',
+                            _class='not-authorized alert alert-block'))
         messages.lock_keys = True
 
         # for "remember me" option
@@ -1876,7 +1877,7 @@ class Auth(AuthAPI):
         # _next variable in the request.
         if next:
             parts = next.split('/')
-            if not ':' in parts[0]:
+            if ':' not in parts[0]:
                 return next
             elif len(parts) > 2 and parts[0].endswith(':') and parts[1:3] == ['', host]:
                 return next
@@ -2008,8 +2009,7 @@ class Auth(AuthAPI):
                 items.append({'name': T('Lost password?'),
                               'href': href('request_reset_password'),
                               'icon': 'icon-lock'})
-            if (self.settings.use_username and not
+            if self.settings.use_username and 'retrieve_username' not in self.settings.actions_disabled:
-                    'retrieve_username' in self.settings.actions_disabled):
                 items.append({'name': T('Forgot username?'),
                               'href': href('retrieve_username'),
                               'icon': 'icon-edit'})
@@ -2181,9 +2181,7 @@ class Auth(AuthAPI):
             current_record.replace('_', ' ').title())
         for table in tables:
             fieldnames = table.fields()
-            if ('id' in fieldnames and
+            if 'id' in fieldnames and 'modified_on' in fieldnames and current_record not in fieldnames:
-                    'modified_on' in fieldnames and
-                    not current_record in fieldnames):
                 table._enable_record_versioning(archive_db=archive_db,
                                                 archive_name=archive_names,
                                                 current_record=current_record,
@@ -2213,7 +2211,8 @@ class Auth(AuthAPI):
             fake_migrate = db._fake_migrate
         settings = self.settings
         settings.enable_tokens = enable_tokens
-        signature_list = super(Auth, self).define_tables(username, signature, migrate, fake_migrate)._table_signature_list
+        signature_list = \
+            super(Auth, self).define_tables(username, signature, migrate, fake_migrate)._table_signature_list
 
         now = current.request.now
         reference_table_user = 'reference %s' % settings.table_user_name
@@ -2360,7 +2359,7 @@ class Auth(AuthAPI):
             if callable(basic_auth_realm):
                 basic_auth_realm = basic_auth_realm()
             elif isinstance(basic_auth_realm, (unicode, str)):
-                basic_realm = unicode(basic_auth_realm)
+                basic_realm = unicode(basic_auth_realm)  # Warning python 3.5 does not have method unicod
             elif basic_auth_realm is True:
                 basic_realm = u'' + current.request.application
             http_401 = HTTP(401, u'Not Authorized', **{'WWW-Authenticate': u'Basic realm="' + basic_realm + '"'})
@@ -2462,7 +2461,7 @@ class Auth(AuthAPI):
                          _href=service + query_sep + "ticket=" + ticket)
             else:
                 redirect(service + query_sep + "ticket=" + ticket)
-        if self.is_logged_in() and not 'renew' in request.vars:
+        if self.is_logged_in() and 'renew' not in request.vars:
             return allow_access()
         elif not self.is_logged_in() and 'gateway' in request.vars:
             redirect(session._cas_service)
@@ -2779,7 +2778,7 @@ class Auth(AuthAPI):
         # If auth.settings.auth_two_factor_enabled it will enable two factor
         # for all the app. Another way to anble two factor is that the user
         # must be part of a group that is called auth.settings.two_factor_authentication_group
-        if user and self.settings.auth_two_factor_enabled == True:
+        if user and self.settings.auth_two_factor_enabled is True:
             session.auth_two_factor_enabled = True
         elif user and self.settings.two_factor_authentication_group:
             role = self.settings.two_factor_authentication_group
@@ -2809,7 +2808,7 @@ class Auth(AuthAPI):
                 # Set the way we generate the code or we send the code. For example using SMS...
                 two_factor_methods = self.settings.two_factor_methods
 
-                if two_factor_methods == []:
+                if not two_factor_methods:
                     # TODO: Add some error checking to handle cases where email cannot be sent
                     self.settings.mailer.send(
                         to=user.email,
@@ -2832,47 +2831,49 @@ class Auth(AuthAPI):
                             hideerror=settings.hideerror):
                 accepted_form = True
 
-                '''
+                """
                 The lists is executed after form validation for each of the corresponding action.
                 For example, in your model:
 
                 In your models copy and paste:
 
-                #Before define tables, we add some extra field to auth_user
+                # Before define tables, we add some extra field to auth_user
                 auth.settings.extra_fields['auth_user'] = [
                     Field('motp_secret', 'password', length=512, default='', label='MOTP Secret'),
                     Field('motp_pin', 'string', length=128, default='', label='MOTP PIN')]
 
-                OFFSET = 60 #Be sure is the same in your OTP Client
+                OFFSET = 60  # Be sure is the same in your OTP Client
 
-                #Set session.auth_two_factor to None. Because the code is generated by external app.
+                # Set session.auth_two_factor to None. Because the code is generated by external app.
                 # This will avoid to use the default setting and send a code by email.
                 def _set_two_factor(user, auth_two_factor):
                     return None
 
                 def verify_otp(user, otp):
-                import time
+                    import time
-                from hashlib import md5
+                    from hashlib import md5
-                epoch_time = int(time.time())
+                    epoch_time = int(time.time())
-                time_start = int(str(epoch_time - OFFSET)[:-1])
+                    time_start = int(str(epoch_time - OFFSET)[:-1])
-                time_end = int(str(epoch_time + OFFSET)[:-1])
+                    time_end = int(str(epoch_time + OFFSET)[:-1])
-                for t in range(time_start - 1, time_end + 1):
+                    for t in range(time_start - 1, time_end + 1):
-                    to_hash = str(t) + user.motp_secret + user.motp_pin
+                        to_hash = str(t) + user.motp_secret + user.motp_pin
-                    hash = md5(to_hash).hexdigest()[:6]
+                        hash = md5(to_hash).hexdigest()[:6]
-                    if otp == hash:
+                        if otp == hash:
-                    return hash
+                        return hash
 
                 auth.settings.auth_two_factor_enabled = True
                 auth.messages.two_factor_comment = "Verify your OTP Client for the code."
-                auth.settings.two_factor_methods = [lambda user, auth_two_factor: _set_two_factor(user, auth_two_factor)]
+                auth.settings.two_factor_methods = [lambda user, 
+                                                           auth_two_factor: _set_two_factor(user, auth_two_factor)]
                 auth.settings.two_factor_onvalidation = [lambda user, otp: verify_otp(user, otp)]
 
-                '''
+                """
-                if self.settings.two_factor_onvalidation != []:
+                if self.settings.two_factor_onvalidation:
 
                     for two_factor_onvalidation in self.settings.two_factor_onvalidation:
                         try:
-                            session.auth_two_factor = two_factor_onvalidation(session.auth_two_factor_user, form.vars['authentication_code'])
+                            session.auth_two_factor = \
+                                two_factor_onvalidation(session.auth_two_factor_user, form.vars['authentication_code'])
                         except:
                             pass
                         else:
@@ -4146,7 +4147,7 @@ class Auth(AuthAPI):
             archive_table = table._db[archive_table_name]
         new_record = {current_record: form.vars.id}
         for fieldname in archive_table.fields:
-            if not fieldname in ['id', current_record]:
+            if fieldname not in ['id', current_record]:
                 if archive_current and fieldname in form.vars:
                     new_record[fieldname] = form.vars[fieldname]
                 elif form.record and fieldname in form.record:
@@ -4956,7 +4957,10 @@ class Service(object):
 
             Then call it with:
 
-                wget --post-data '{"jsonrpc": "2.0", "id": 1, "method": "myfunction", "params": {"a": 1, "b": 2}}' http://..../app/default/call/jsonrpc2
+                wget --post-data '{"jsonrpc": "2.0",
+                                   "id": 1,
+                                   "method": "myfunction",
+                                   "params": {"a": 1, "b": 2}}' http://..../app/default/call/jsonrpc2
 
         """
         self.jsonrpc2_procedures[f.__name__] = f
@@ -5527,15 +5531,15 @@ def prettydate(d, T=lambda x: x, utc=False):
     else:
         suffix = ' ago'
     if dt.days >= 2 * 365:
-        return T('%d years' + suffix) % int(dt.days / 365)
+        return T('%d years' + suffix) % int(dt.days // 365)
     elif dt.days >= 365:
         return T('1 year' + suffix)
     elif dt.days >= 60:
-        return T('%d months' + suffix) % int(dt.days / 30)
+        return T('%d months' + suffix) % int(dt.days // 30)
     elif dt.days >= 27:  # 4 weeks ugly
         return T('1 month' + suffix)
     elif dt.days >= 14:
-        return T('%d weeks' + suffix) % int(dt.days / 7)
+        return T('%d weeks' + suffix) % int(dt.days // 7)
     elif dt.days >= 7:
         return T('1 week' + suffix)
     elif dt.days > 1:
@@ -5543,11 +5547,11 @@ def prettydate(d, T=lambda x: x, utc=False):
     elif dt.days == 1:
         return T('1 day' + suffix)
     elif dt.seconds >= 2 * 60 * 60:
-        return T('%d hours' + suffix) % int(dt.seconds / 3600)
+        return T('%d hours' + suffix) % int(dt.seconds // 3600)
     elif dt.seconds >= 60 * 60:
         return T('1 hour' + suffix)
     elif dt.seconds >= 2 * 60:
-        return T('%d minutes' + suffix) % int(dt.seconds / 60)
+        return T('%d minutes' + suffix) % int(dt.seconds // 60)
     elif dt.seconds >= 60:
         return T('1 minute' + suffix)
     elif dt.seconds > 1:
@@ -5600,12 +5604,12 @@ class PluginManager(object):
 
         where the plugin is used::
 
-            >>> print plugins.me.param1
+            >>> print(plugins.me.param1)
             3
-            >>> print plugins.me.param2
+            >>> print(plugins.me.param2)
             6
             >>> plugins.me.param3 = 8
-            >>> print plugins.me.param3
+            >>> print(plugins.me.param3)
             8
 
         Here are some tests::
@@ -5613,25 +5617,25 @@ class PluginManager(object):
             >>> a=PluginManager()
             >>> a.x=6
             >>> b=PluginManager('check')
-            >>> print b.x
+            >>> print(b.x)
             6
             >>> b=PluginManager() # reset settings
-            >>> print b.x
+            >>> print(b.x)
             <Storage {}>
             >>> b.x=7
-            >>> print a.x
+            >>> print(a.x)
             7
             >>> a.y.z=8
-            >>> print b.y.z
+            >>> print(b.y.z)
             8
             >>> test_thread_separation()
             5
             >>> plugins=PluginManager('me',db='mydb')
-            >>> print plugins.me.db
+            >>> print(plugins.me.db)
             mydb
-            >>> print 'me' in plugins
+            >>> print('me' in plugins)
             True
-            >>> print plugins.me.installed
+            >>> print(plugins.me.installed)
             True
 
     """
@@ -5780,7 +5784,7 @@ class Expose(object):
             return os.path.realpath(f)
 
     def issymlink_out(self, f):
-        "True if f is a symlink and is pointing outside of self.base"
+        """True if f is a symlink and is pointing outside of self.base"""
         return os.path.islink(f) and not self.in_base(f)
 
     @staticmethod

gluon/utils.py

@@ -156,12 +156,12 @@ def get_digest(value):
         raise ValueError("Invalid digest algorithm: %s" % value)
 
 DIGEST_ALG_BY_SIZE = {
-    128 / 4: 'md5',
+    128 // 4: 'md5',
-    160 / 4: 'sha1',
+    160 // 4: 'sha1',
-    224 / 4: 'sha224',
+    224 // 4: 'sha224',
-    256 / 4: 'sha256',
+    256 // 4: 'sha256',
-    384 / 4: 'sha384',
+    384 // 4: 'sha384',
-    512 / 4: 'sha512',
+    512 // 4: 'sha512',
 }
 
 

gluon/validators.py

@@ -21,7 +21,8 @@ import struct
 import decimal
 import unicodedata
 
-from gluon._compat import StringIO, long, basestring, unicodeT, to_unicode, urllib_unquote, unichr, to_bytes, PY2, to_unicode, to_native, string_types, urlparse
+from gluon._compat import StringIO, long, basestring, unicodeT, to_unicode, urllib_unquote, unichr, to_bytes, PY2, \
+    to_unicode, to_native, string_types, urlparse
 from gluon.utils import simple_hash, web2py_uuid, DIGEST_ALG_BY_SIZE
 from pydal.objects import Field, FieldVirtual, FieldMethod
 from functools import reduce
@@ -452,10 +453,10 @@ class IS_IN_SET(Validator):
         if not self.labels:
             items = [(k, k) for (i, k) in enumerate(self.theset)]
         else:
-            items = [(k, self.labels[i]) for (i, k) in enumerate(self.theset)]
+            items = [(k, list(self.labels)[i]) for (i, k) in enumerate(self.theset)]
         if self.sort:
             items.sort(key=lambda o: str(o[1]).upper())
-        if zero and not self.zero is None and not self.multiple:
+        if zero and self.zero is not None and not self.multiple:
             items.insert(0, ('', self.zero))
         return items
 
@@ -823,7 +824,7 @@ class IS_INT_IN_RANGE(Validator):
 
 def str2dec(number):
     s = str(number)
-    if not '.' in s:
+    if '.' not in s:
         s += '.00'
     else:
         s += '0' * (2 - len(s.split('.')[1]))
@@ -1213,7 +1214,7 @@ class IS_EMAIL(Validator):
                 domain_encoded = to_unicode(domain).encode('idna').decode('ascii')
                 match_domain = self.domain_regex.match(domain_encoded)
 
-            match = (match_body != None) and (match_domain != None)
+            match = (match_body is not None) and (match_domain is not None)
         except (TypeError, UnicodeError):
             # Value may not be a string where we can look for matches.
             # Example: we're calling ANY_OF formatter and IS_EMAIL is asked to validate a date.
@@ -1247,7 +1248,7 @@ class IS_LIST_OF_EMAILS(object):
         f = IS_EMAIL()
         for email in self.split_emails.findall(value):
             error = f(email)[1]
-            if error and not email in bad_emails:
+            if error and email not in bad_emails:
                 bad_emails.append(email)
         if not bad_emails:
             return (value, None)
@@ -1461,9 +1462,9 @@ def unicode_to_ascii_authority(authority):
         if label:
             asciiLabels.append(to_native(encodings.idna.ToASCII(label)))
         else:
-             # encodings.idna.ToASCII does not accept an empty string, but
+            # encodings.idna.ToASCII does not accept an empty string, but
-             # it is necessary for us to allow for empty labels so that we
+            # it is necessary for us to allow for empty labels so that we
-             # don't modify the URL
+            # don't modify the URL
             asciiLabels.append('')
     # RFC 3490, Section 4, Step 5
     return str(reduce(lambda x, y: x + unichr(0x002E) + y, asciiLabels))
@@ -1527,13 +1528,17 @@ def unicode_to_ascii_url(url, prepend_scheme):
     if prepended:
         scheme = ''
 
-    unparsed = urlparse.urlunparse((scheme, unicode_to_ascii_authority(authority), escape_unicode(path), '', escape_unicode(query), str(fragment)))
+    unparsed = urlparse.urlunparse((scheme,
+                                    unicode_to_ascii_authority(authority),
+                                    escape_unicode(path),
+                                    '',
+                                    escape_unicode(query),
+                                    str(fragment)))
     if unparsed.startswith('//'):
-        unparsed = unparsed[2:] # Remove the // urlunparse puts in the beginning
+        unparsed = unparsed[2:]  # Remove the // urlunparse puts in the beginning
     return unparsed
 
 
-
 class IS_GENERIC_URL(Validator):
     """
     Rejects a URL string if any of the following is true:
@@ -2622,7 +2627,7 @@ class ANY_OF(Validator):
     def __call__(self, value):
         for validator in self.subs:
             value, error = validator(value)
-            if error == None:
+            if error is None:
                 break
         return value, error
 
@@ -2762,7 +2767,7 @@ class LazyCrypt(object):
         else:
             digest_alg, key = self.crypt.digest_alg, ''
         if self.crypt.salt:
-            if self.crypt.salt == True:
+            if self.crypt.salt is True:
                 salt = str(web2py_uuid()).replace('-', '')[-16:]
             else:
                 salt = self.crypt.salt
@@ -2847,7 +2852,7 @@ class CRYPT(object):
     Supports standard algorithms
 
         >>> for alg in ('md5','sha1','sha256','sha384','sha512'):
-        ...     print str(CRYPT(digest_alg=alg,salt=True)('test')[0])
+        ...     print(str(CRYPT(digest_alg=alg,salt=True)('test')[0]))
         md5$...$...
         sha1$...$...
         sha256$...$...
@@ -2859,13 +2864,13 @@ class CRYPT(object):
     Supports for pbkdf2
 
         >>> alg = 'pbkdf2(1000,20,sha512)'
-        >>> print str(CRYPT(digest_alg=alg,salt=True)('test')[0])
+        >>> print(str(CRYPT(digest_alg=alg,salt=True)('test')[0]))
         pbkdf2(1000,20,sha512)$...$...
 
     An optional hmac_key can be specified and it is used as salt prefix
 
         >>> a = str(CRYPT(digest_alg='md5',key='mykey',salt=True)('test')[0])
-        >>> print a
+        >>> print(a)
         md5$...$...
 
     Even if the algorithm changes the hash can still be validated