sync with master branch

.travis.yml

@@ -4,11 +4,18 @@ sudo: required
 
 cache: pip
 
+dist: "trusty"
+
 python:
   - '2.7'
-  - 'pypy'
   - '3.5'
   - '3.6'
+  - 'pypy-5.3.1'
+  - 'pypy3.5-5.7.1-beta'
+
+matrix:
+  allow_failures:
+    - python: 'pypy3.5-5.7.1-beta'
 
 install:
   - pip install -e .
@@ -33,3 +40,6 @@ notifications:
 
 addons:
   postgresql: "9.4"
+  apt:
+    packages:
+      - postgresql-9.4-postgis-2.3

CHANGELOG

@@ -1,8 +1,30 @@
-
-## 2.15.x
-- web2py does not support python 2.6 anymore
-- py3.5 syntax compatible (see #1353 for details)
-- dropped web shell from admin
+## 2.15.1-4
+- pydal 17.08
+- dropped support for python 2.6
+- dropped web shell
+- experimental python 3 support
+- experimental authapi for service login
+- allow ajax file uploads
+- more tests
+- more pep8 compliance
+- d3.js model visulization
+- improved scheduler
+- is_email support for internationalized Domain Names
+- improved used of cookies with CookieJar
+- SQLFORM.grid(showblobs=True)
+- import JS events (added w2p.componentBegin event)
+- added support for CASv3
+- allow first_name and last_name placeholders in verify_email message
+- added three-quote support in markmin
+- updated pg8000 driver (but we still recommend psycopg2)
+- compiled views use . separator not _ separator (must recompile code)
+- better serbian, french, and catalan translations
+- speed improvements (refactor of compileapp and pyc caching)
+- removed web shell (never worked as intended)
+- allow Expose(..., follow_symlink_out=False).
+- Updated fpdf to latest version
+- JWT support
+- import fabfile for remote deployment
 - scheduler new feature: you can now specify intervals with cron
 - gluon/* removed from sys.path. Applications relying on statements like e.g.
   "from storage import Storage"
@@ -10,8 +32,18 @@
   "from gluon.storage import Storage"
 - tests can only be run with the usual web2py.py --run_system_tests OR with
   python -m unittest -v gluon.tests on the root dir
-- updated pymysql driver
-
+- jQuery 3.2.1
+- PyDAL 17.07 including:
+  allow jsonb support for postgres
+  correctly configure adapters that need connection for configuration
+  better caching
+  updated IMAP adapter methods to new API
+  experimental suport for joinable subselects
+  improved Teradata support
+  improved mongodb support
+  overall refactoring
+  experimental support for Google Cloud SQL v2
+  new pymysql driver
 
 ## 2.14.6
 

Makefile

@@ -32,7 +32,7 @@ update:
 	echo "remember that pymysql was tweaked"
 src:
 	### Use semantic versioning
-	echo 'Version 2.14.6-stable+timestamp.'`date +%Y.%m.%d.%H.%M.%S` > VERSION
+	echo 'Version 2.15.4-stable+timestamp.'`date +%Y.%m.%d.%H.%M.%S` > VERSION
 	### rm -f all junk files
 	make clean
 	### clean up baisc apps
@@ -54,7 +54,7 @@ src:
 	### build web2py_src.zip
 	echo '' > NEWINSTALL
 	mv web2py_src.zip web2py_src_old.zip | echo 'no old'
-	cd ..; zip -r web2py/web2py_src.zip web2py/web2py.py web2py/anyserver.py web2py/gluon/* web2py/extras/* web2py/handlers/* web2py/examples/* web2py/README.markdown  web2py/LICENSE web2py/CHANGELOG web2py/NEWINSTALL web2py/VERSION web2py/MANIFEST.in web2py/scripts/*.sh web2py/scripts/*.py web2py/applications/admin web2py/applications/examples/ web2py/applications/welcome web2py/applications/__init__.py web2py/site-packages/__init__.py web2py/gluon/tests/*.sh web2py/gluon/tests/*.py
+	cd ..; zip -r web2py/web2py_src.zip web2py/web2py.py web2py/anyserver.py web2py/fabfile.py web2py/gluon/* web2py/extras/* web2py/handlers/* web2py/examples/* web2py/README.markdown  web2py/LICENSE web2py/CHANGELOG web2py/NEWINSTALL web2py/VERSION web2py/MANIFEST.in web2py/scripts/*.sh web2py/scripts/*.py web2py/applications/admin web2py/applications/examples/ web2py/applications/welcome web2py/applications/__init__.py web2py/site-packages/__init__.py web2py/gluon/tests/*.sh web2py/gluon/tests/*.py
 
 mdp:
 	make src
@@ -97,6 +97,8 @@ win:
 	cp -r applications/welcome ../web2py_win/web2py/applications
 	cp -r applications/examples ../web2py_win/web2py/applications
 	cp applications/__init__.py ../web2py_win/web2py/applications
+	# per https://github.com/web2py/web2py/issues/1716
+	mv ../web2py_win/web2py/_ssl.pyd ../web2py_win/web2py/_ssl.pyd.legacy | echo 'done'
 	cd ../web2py_win; zip -r web2py_win.zip web2py
 	mv ../web2py_win/web2py_win.zip .
 run:

VERSION

@@ -1 +1 @@
-Version 2.14.6-stable+timestamp.2016.05.09.19.18.48
+Version 2.15.4-stable+timestamp.2017.09.01.22.38.25

applications/admin/controllers/appadmin.py

@@ -12,11 +12,6 @@ import gluon.contenttype
 import gluon.fileutils
 from gluon._compat import iteritems
 
-try:
-    import pygraphviz as pgv
-except ImportError:
-    pgv = None
-
 is_gae = request.env.web2py_runtime_gae or False
 
 # ## critical --- make a copy of the environment
@@ -565,57 +560,6 @@ def table_template(table):
                                           _cellborder=0, _cellspacing=0)
                              ).xml()
 
-
-def bg_graph_model():
-    graph = pgv.AGraph(layout='dot',  directed=True,  strict=False,  rankdir='LR')
-
-    subgraphs = dict()
-    for tablename in db.tables:
-        if hasattr(db[tablename],'_meta_graphmodel'):
-            meta_graphmodel = db[tablename]._meta_graphmodel
-        else:
-            meta_graphmodel = dict(group=request.application, color='#ECECEC')
-
-        group = meta_graphmodel['group'].replace(' ', '')
-        if group not in subgraphs:
-            subgraphs[group] = dict(meta=meta_graphmodel, tables=[])
-        subgraphs[group]['tables'].append(tablename)
-
-        graph.add_node(tablename, name=tablename, shape='plaintext',
-                       label=table_template(tablename))
-
-    for n, key in enumerate(subgraphs.iterkeys()):
-        graph.subgraph(nbunch=subgraphs[key]['tables'],
-                    name='cluster%d' % n,
-                    style='filled',
-                    color=subgraphs[key]['meta']['color'],
-                    label=subgraphs[key]['meta']['group'])
-
-    for tablename in db.tables:
-        for field in db[tablename]:
-            f_type = field.type
-            if isinstance(f_type,str) and (
-                f_type.startswith('reference') or
-                f_type.startswith('list:reference')):
-                referenced_table = f_type.split()[1].split('.')[0]
-                n1 = graph.get_node(tablename)
-                n2 = graph.get_node(referenced_table)
-                graph.add_edge(n1, n2, color="#4C4C4C", label='')
-
-    graph.layout()
-    if not request.args:
-        response.headers['Content-Type'] = 'image/png'
-        return graph.draw(format='png', prog='dot')
-    else:
-        response.headers['Content-Disposition']='attachment;filename=graph.%s'%request.args(0)
-        if request.args(0) == 'dot':
-            return graph.string()
-        else:
-            return graph.draw(format=request.args(0), prog='dot')
-
-def graph_model():
-    return dict(databases=databases, pgv=pgv)
-
 def manage():
     tables = manager_action['tables']
     if isinstance(tables[0], str):
@@ -700,3 +644,51 @@ def hooks():
             ul_t.append(UL([LI(A(f['funcname'], _class="editor_filelink", _href=f['url']if 'url' in f else None, **{'_data-lineno':f['lineno']-1})) for f in op['functions']]))
         ul_main.append(ul_t)
     return ul_main
+
+
+# ##########################################################
+# d3 based model visualizations
+# ###########################################################
+
+def d3_graph_model():
+    """ See https://www.facebook.com/web2py/posts/145613995589010 from Bruno Rocha
+    and also the app_admin bg_graph_model function
+    
+    Create a list of table dicts, called "nodes"
+    """
+    
+    nodes = []
+    links = []
+
+    for database in databases:
+        db = eval_in_global_env(database)
+        for tablename in db.tables:
+            fields = []
+            for field in db[tablename]:
+                f_type = field.type
+                if not isinstance(f_type,str):
+                    disp = ' '
+                elif f_type == 'string':
+                    disp =  field.length
+                elif f_type == 'id':
+                    disp =  "PK"
+                elif f_type.startswith('reference') or \
+                    f_type.startswith('list:reference'):
+                    disp = "FK"
+                else:
+                    disp = ' '
+                fields.append(dict(name= field.name, type=field.type, disp = disp))
+
+                if isinstance(f_type,str) and (
+                    f_type.startswith('reference') or
+                    f_type.startswith('list:reference')):
+                    referenced_table = f_type.split()[1].split('.')[0]
+
+                    links.append(dict(source=tablename, target = referenced_table))
+
+            nodes.append(dict(name=tablename, type="table", fields = fields))
+
+    # d3 v4 allows individual modules to be specified.  The complete d3 library is included below.
+    response.files.append(URL('admin','static','js/d3.min.js'))
+    response.files.append(URL('admin','static','js/d3_graph.js'))
+    return dict(databases=databases, nodes=nodes, links=links)

applications/admin/controllers/debug.py

@@ -6,6 +6,7 @@ import gluon.validators
 import code
 from gluon.debug import communicate, web_debugger, dbg_debugger
 from gluon._compat import thread
+from gluon.fileutils import open_file
 import pydoc
 
 
@@ -54,7 +55,7 @@ def interact():
     if filename:
         # prevent IOError 2 on some circuntances (EAFP instead of os.access)
         try:
-            lines = open(filename).readlines()
+            lines = open_file(filename, 'r').readlines()
         except:
             lines = ""
         lines = dict([(i + 1, l) for (i, l) in enumerate(

applications/admin/controllers/default.py

@@ -16,6 +16,7 @@ from gluon.tools import Config
 from gluon.compileapp import find_exposed_functions
 from glob import glob
 from gluon._compat import iteritems, PY2, pickle, xrange, urlopen, to_bytes, StringIO, to_native
+import gluon.rewrite
 import shutil
 import platform
 
@@ -249,6 +250,7 @@ def site():
                 db.app.insert(name=appname, owner=auth.user.id)
             log_progress(appname)
             session.flash = T('new application "%s" created', appname)
+            gluon.rewrite.load()
             redirect(URL('design', args=appname))
         else:
             session.flash = \
@@ -266,6 +268,7 @@ def site():
                 new_repo = git.Repo.clone_from(form_update.vars.url, target)
                 session.flash = T('new application "%s" imported',
                                   form_update.vars.name)
+                gluon.rewrite.load()
             except git.GitCommandError as err:
                 session.flash = T('Invalid git repository specified.')
             redirect(URL(r=request))
@@ -302,6 +305,7 @@ def site():
             log_progress(appname)
             session.flash = T(msg, dict(appname=appname,
                                         digest=md5_hash(installed)))
+            gluon.rewrite.load()
         else:
             msg = 'unable to install application "%(appname)s"'
             session.flash = T(msg, dict(appname=form_update.vars.name))
@@ -1861,7 +1865,6 @@ def user():
 
 def reload_routes():
     """ Reload routes.py """
-    import gluon.rewrite
     gluon.rewrite.load()
     redirect(URL('site'))
 

applications/admin/models/access.py

@@ -52,7 +52,7 @@ def verify_password(password):
     if DEMO_MODE:
         ret = True
     elif not _config.get('password'):
-        ret - False
+        ret = False
     elif _config['password'].startswith('pam_user:'):
         session.pam_user = _config['password'][9:].strip()
         import gluon.contrib.pam

applications/admin/static/css/d3_graph.css

@@ -0,0 +1,33 @@
+    .node {fill: steelblue;
+           stroke: #636363;
+           stroke-width: 1px;}
+
+    .auth {fill: lightgrey;}
+
+    .table {r: 10;}
+
+    .link {stroke: #bbbbbb;
+           stroke-width: 2px;}
+    td {padding: 4px;}
+
+    div.tooltip {
+       position: absolute;
+       text-align: left;
+       /* width: 140px; */
+       /* height: 28px;*/
+       padding: 0px 5px 0px 5px;
+       padding-top: 0px;
+       font: 12px sans-serif;
+       background: #fff7bc;
+       border: solid 1px #aaa;
+       border-radius: 6px;
+       pointer-events: none;}
+
+    h5 { font: 14px sans-serif;
+         background : #ec7014;
+         color: #ffffe5; 
+         padding: 5px 2px 5px 2px;
+         margin-top: 1px;}
+    path {
+           fill: #aaaaaa;}
+

applications/admin/static/js/d3_graph.js

@@ -0,0 +1,181 @@
+function d3_graph() {
+
+// Some reference links:
+//   How to get link ids instead of index
+//     http://stackoverflow.com/questions/23986466/d3-force-layout-linking-nodes-by-name-instead-of-index
+//   embedding web2py in d3
+//     http://stackoverflow.com/questions/34326343/embedding-d3-js-graph-in-a-web2py-bootstrap-page
+
+// nodes and links are defined in appadmin.html <script>
+
+
+    var edges = [];
+
+    links.forEach(function(e) {
+        var sourceNode = nodes.filter(function(n) {
+            return n.name === e.source;
+        })[0],
+            targetNode = nodes.filter(function(n) {
+            return n.name === e.target;
+        })[0];
+
+        edges.push({
+            source: sourceNode,
+            target: targetNode,
+            value: 1});
+
+    });
+
+    edges.forEach(function(e) {
+
+        if (!e.source["linkcount"]) e.source["linkcount"] = 0;
+        if (!e.target["linkcount"]) e.target["linkcount"] = 0;
+
+        e.source["linkcount"]++;
+        e.target["linkcount"]++;
+    });
+
+    //var width = 960, height = 600;
+    var height = window.innerHeight|| docEl.clientHeight|| bodyEl.clientHeight;
+    var width = window.innerWidth || docEl.clientWidth || bodyEl.clientWidth;
+    var svg = d3.select("#vis").append("svg")
+            .attr("width", width)
+            .attr("height", height);
+
+       // updated for d3 v4.
+    var simulation = d3.forceSimulation()
+            .force("link", d3.forceLink().id(function(d) { return d.id; }))
+            .force("charge", d3.forceManyBody().strength(strength))
+            .force("center", d3.forceCenter(width / 2, height / 2))
+            .force("collision", d3.forceCollide(35));
+
+    // Node charge strength.  Repel strength greater for less links.
+    //function strength(d) { return -50/d["linkcount"] ; }
+    function strength(d) { return -25 ; }  
+    
+    // Link distance.  Distance increases with number of links at source and target
+    function distance(d) { return (60 + (d.source["linkcount"] * d.target["linkcount"])) ; }
+    
+    // Link strength.  Strength is less for highly connected nodes (move towards target dist)
+    function strengthl(d) { return 5/(d.source["linkcount"] + d.target["linkcount"]) ; }
+
+    simulation
+        .nodes(nodes)
+        .on("tick", tick);
+
+    simulation.force("link")
+        .links(edges)
+        .distance(distance)
+        .strength(strengthl);
+
+    // build the arrow.
+    svg.append("svg:defs").selectAll("marker")
+        .data(["end"])      // Different link/path types can be defined here
+        .enter().append("svg:marker")    // This section adds in the arrows
+        .attr("id", String)
+        .attr("viewBox", "0 -5 10 10")
+        .attr("refX", 25)   // Moves the arrow head out, allow for radius
+        .attr("refY", 0)   // -1.5
+        .attr("markerWidth", 6)
+        .attr("markerHeight", 6)
+        .attr("orient", "auto")
+        .append("svg:path")
+        .attr("d", "M0,-5L10,0L0,5");
+
+    var link = svg.selectAll('.link')
+        .data(edges)
+        .enter().append('line')
+        .attr("class", "link")
+        .attr("marker-end", "url(#end)");
+
+    var node = svg.selectAll(".node")
+        .data(nodes)
+        .enter().append("g")
+        .attr("class", function(d) { return "node " + d.type;})
+        .attr('transform', function(d) {
+            return "translate(" + d.x + "," + d.y + ")"})
+        .classed("auth", function(d) { return (d.name.startsWith("auth") ? true : false);});
+
+    node.call(d3.drag()
+            .on("start", dragstarted)
+            .on("drag", dragged)
+            .on("end", dragended));
+
+    // add the nodes
+    node.append('circle')
+        .attr('r', 16)
+        ;
+
+    // add text
+    node.append("text")
+        .attr("x", 12)
+        .attr("dy", "-1.1em")
+        .text(function(d) {return d.name;});
+
+    node.on("mouseover", function(d) {
+
+        var g = d3.select(this);  // the node (table)
+
+        // tooltip
+
+        var fields = d.fields;
+        var fieldformat = "<TABLE>";
+        fields.forEach(function(d) {
+            fieldformat += "<TR><TD><B>"+ d.name+"</B></TD><TD>"+ d.type+"</TD><TD>"+ d.disp+"</TD></TR>";
+        });
+        fieldformat += "</TABLE>";
+        var tiplength = d.fields.length;
+
+        // Define 'div' for tooltips
+        var div = d3.select("body").append("div")  // declare the tooltip div
+	        .attr("class", "tooltip")              // apply the 'tooltip' class
+            .style("opacity", 0)
+            .html('<h5>' + d.name + '</h5>' + fieldformat)
+            .style("left", 20 + (d3.event.pageX) + "px")// or just (d.x + 50 + "px")
+            .style("top", tooltop(tiplength))// or ...
+            .transition()
+            .duration(800)
+            .style("opacity", 0.9);
+        });
+
+        function tooltop(tiplength) {
+           //aim to ensure tooltip is fully visible whenver possible
+           return (Math.max(d3.event.pageY - 20 - (tiplength * 14),0)) + "px"
+        }
+
+        node.on("mouseout", function(d) {
+            d3.select("body").select('div.tooltip').remove();
+    });
+
+    // instead of waiting for force to end with :     force.on('end', function()
+    // use .on("tick",  instead.  Here is the tick function
+    function tick() {
+        node.attr('transform', function(d) {
+            d.x = Math.max(30, Math.min(width - 16, d.x));
+            d.y = Math.max(30, Math.min(height - 16, d.y));
+            return "translate(" + d.x + "," + d.y + ")"; });
+
+        link.attr('x1', function(d) {return d.source.x;})
+            .attr('y1', function(d) {return d.source.y;})
+            .attr('x2', function(d) {return d.target.x;})
+            .attr('y2', function(d) {return d.target.y;});
+    };
+
+    function dragstarted(d) {
+        if (!d3.event.active) simulation.alphaTarget(0.3).restart();
+        d.fx = d.x;
+        d.fy = d.y;
+    };
+
+    function dragged(d) {
+        d.fx = d3.event.x;
+        d.fy = d3.event.y;
+    };
+
+    function dragended(d) {
+        if (!d3.event.active) simulation.alphaTarget(0);
+        d.fx = null;
+        d.fy = null;
+    };
+
+};

applications/admin/static/js/web2py.js

@@ -38,8 +38,12 @@
             if (value > 0) $('#' + id).hide().fadeIn('slow');
             else $('#' + id).show().fadeOut('slow');
         },
-        ajax: function (u, s, t) {
+        ajax: function (u, s, t, options) {
             /*simple ajax function*/
+
+            // set options default value
+            options = typeof options !== 'undefined' ? options : {};
+
             var query = '';
             if (typeof s == 'string') {
                 var d = $(s).serialize();
@@ -59,18 +63,44 @@
                     query = pcs.join('&');
                 }
             }
-            $.ajax({
+
+            // default success action
+            var success_function = function (msg) {
+                if (t) {
+                    if (t == ':eval') eval(msg);
+                    else if (typeof t == 'string') $('#' + t).html(msg);
+                    else t(msg);
+                }
+            };
+
+            // declare success actions as array
+            var success = [success_function];
+
+            // add user success actions
+            if ($.isArray(options.done)){
+                success = $.merge(success, options.done);
+            } else {
+                success.push(options.done);
+            }
+
+            // default jquery ajax options
+            var ajax_options = {
                 type: 'POST',
                 url: u,
                 data: query,
-                success: function (msg) {
-                    if (t) {
-                        if (t == ':eval') eval(msg);
-                        else if (typeof t == 'string') $('#' + t).html(msg);
-                        else t(msg);
-                    }
-                }
-            });
+                success: success
+            };
+
+            //remove custom "done" option if exists
+            delete options.done;
+
+            // merge default ajax options with user custom options
+            for (var attrname in options) {
+                    ajax_options[attrname] = options[attrname];
+            }
+
+            // call ajax function
+            $.ajax(ajax_options);
         },
         ajax_fields: function (target) {
             /*

applications/admin/views/appadmin.html

@@ -233,30 +233,22 @@
 <div class="clear"></div>
 {{pass}}
 
-{{if request.function=='graph_model':}}
+{{if request.function=='d3_graph_model':}}
 <h2>{{=T("Graph Model")}}</h2>
-  {{if not pgv:}}
-    {{=T('pygraphviz library not found')}}
-  {{elif not databases:}}
+  {{if not databases:}}  
     {{=T("No databases in this application")}}
-  {{else:}}	   
-    <div class="btn-group">
-      <a class="btn dropdown-toggle" data-toggle="dropdown" href="#">
-        <i class="icon-download"></i> {{=T('Save model as...')}}
-        <span class="caret"></span>
-      </a>
-      <ul class="dropdown-menu">
-          <li><a href="{{=URL('appadmin', 'bg_graph_model', args=['png'])}}">png</a></li>
-          <li><a href="{{=URL('appadmin', 'bg_graph_model', args=['svg'])}}">svg</a></li>
-          <li><a href="{{=URL('appadmin', 'bg_graph_model', args=['pdf'])}}">pdf</a></li>
-          <li><a href="{{=URL('appadmin', 'bg_graph_model', args=['ps'])}}">ps</a></li>
-          <li><a href="{{=URL('appadmin', 'bg_graph_model', args=['dot'])}}">dot</a></li>
-      </ul>
-    </div>
-    <br />
-    {{=IMG(_src=URL('appadmin', 'bg_graph_model'))}}
+  {{else:}}    
+    <div id="vis"></div>
+      <link rel="stylesheet" href="{{=URL('admin','static','css/d3_graph.css')}}"/>
+      <script>
+        // Define the d3 input data
+        {{from gluon.serializers import json }}
+        var nodes = {{=XML(json(nodes))}};
+        var links = {{=XML(json(links))}};
+        d3_graph();
+      </script>
   {{pass}}
-{{pass}}
+{{pass}}  
 
 {{if request.function == 'manage':}}
 <h2>{{=heading}}</h2>

applications/admin/views/default/design.html

@@ -105,7 +105,9 @@ def deletefile(arglist, vars={}):
         {{if os.access(os.path.join(request.folder,'..',app,'databases','sql.log'),os.R_OK):}}
             {{=button(URL('peek/%s/databases/sql.log'%app), 'sql.log')}}
         {{pass}}
-                {{=button(URL(a=app, c='appadmin',f='graph_model'), T('graph model'))}}
+        {{if os.access(os.path.join(request.folder,'..','admin','static','js','d3_graph.js'),os.R_OK):}}
+                {{=button(URL(a=app, c='appadmin',f='d3_graph_model'), T('graph model'))}}
+        {{pass}}
     </div>
     <ul class="unstyled act_edit">
         {{for m in models:}}

applications/admin/views/default/site.html

@@ -46,7 +46,7 @@
       </td>
       <td>
         <div class="btn-group">
-          <a class="btn dropdown-toggle" data-toggle="dropdown" href="#">
+          <a class="btn dropdown-toggle" data-toggle="dropdown">
         {{=T('Manage')}}
         <span class="caret"></span>
           </a>

applications/admin/views/generic.html

@@ -7,7 +7,7 @@ It is used as default when a view is not provided for your controllers
 """}}
 <h2>{{=' '.join(x.capitalize() for x in request.function.split('_'))}}</h2>
 {{if len(response._vars)==1:}}
-{{=BEAUTIFY(response._vars.values()[0])}}
+{{=BEAUTIFY(response._vars[next(iter(response._vars))])}}
 {{elif len(response._vars)>1:}}
 {{=BEAUTIFY(response._vars)}}
 {{pass}}

applications/examples/controllers/appadmin.py

@@ -12,11 +12,6 @@ import gluon.contenttype
 import gluon.fileutils
 from gluon._compat import iteritems
 
-try:
-    import pygraphviz as pgv
-except ImportError:
-    pgv = None
-
 is_gae = request.env.web2py_runtime_gae or False
 
 # ## critical --- make a copy of the environment
@@ -565,57 +560,6 @@ def table_template(table):
                                           _cellborder=0, _cellspacing=0)
                              ).xml()
 
-
-def bg_graph_model():
-    graph = pgv.AGraph(layout='dot',  directed=True,  strict=False,  rankdir='LR')
-
-    subgraphs = dict()
-    for tablename in db.tables:
-        if hasattr(db[tablename],'_meta_graphmodel'):
-            meta_graphmodel = db[tablename]._meta_graphmodel
-        else:
-            meta_graphmodel = dict(group=request.application, color='#ECECEC')
-
-        group = meta_graphmodel['group'].replace(' ', '')
-        if group not in subgraphs:
-            subgraphs[group] = dict(meta=meta_graphmodel, tables=[])
-        subgraphs[group]['tables'].append(tablename)
-
-        graph.add_node(tablename, name=tablename, shape='plaintext',
-                       label=table_template(tablename))
-
-    for n, key in enumerate(subgraphs.iterkeys()):
-        graph.subgraph(nbunch=subgraphs[key]['tables'],
-                    name='cluster%d' % n,
-                    style='filled',
-                    color=subgraphs[key]['meta']['color'],
-                    label=subgraphs[key]['meta']['group'])
-
-    for tablename in db.tables:
-        for field in db[tablename]:
-            f_type = field.type
-            if isinstance(f_type,str) and (
-                f_type.startswith('reference') or
-                f_type.startswith('list:reference')):
-                referenced_table = f_type.split()[1].split('.')[0]
-                n1 = graph.get_node(tablename)
-                n2 = graph.get_node(referenced_table)
-                graph.add_edge(n1, n2, color="#4C4C4C", label='')
-
-    graph.layout()
-    if not request.args:
-        response.headers['Content-Type'] = 'image/png'
-        return graph.draw(format='png', prog='dot')
-    else:
-        response.headers['Content-Disposition']='attachment;filename=graph.%s'%request.args(0)
-        if request.args(0) == 'dot':
-            return graph.string()
-        else:
-            return graph.draw(format=request.args(0), prog='dot')
-
-def graph_model():
-    return dict(databases=databases, pgv=pgv)
-
 def manage():
     tables = manager_action['tables']
     if isinstance(tables[0], str):
@@ -700,3 +644,51 @@ def hooks():
             ul_t.append(UL([LI(A(f['funcname'], _class="editor_filelink", _href=f['url']if 'url' in f else None, **{'_data-lineno':f['lineno']-1})) for f in op['functions']]))
         ul_main.append(ul_t)
     return ul_main
+
+
+# ##########################################################
+# d3 based model visualizations
+# ###########################################################
+
+def d3_graph_model():
+    """ See https://www.facebook.com/web2py/posts/145613995589010 from Bruno Rocha
+    and also the app_admin bg_graph_model function
+    
+    Create a list of table dicts, called "nodes"
+    """
+    
+    nodes = []
+    links = []
+
+    for database in databases:
+        db = eval_in_global_env(database)
+        for tablename in db.tables:
+            fields = []
+            for field in db[tablename]:
+                f_type = field.type
+                if not isinstance(f_type,str):
+                    disp = ' '
+                elif f_type == 'string':
+                    disp =  field.length
+                elif f_type == 'id':
+                    disp =  "PK"
+                elif f_type.startswith('reference') or \
+                    f_type.startswith('list:reference'):
+                    disp = "FK"
+                else:
+                    disp = ' '
+                fields.append(dict(name= field.name, type=field.type, disp = disp))
+
+                if isinstance(f_type,str) and (
+                    f_type.startswith('reference') or
+                    f_type.startswith('list:reference')):
+                    referenced_table = f_type.split()[1].split('.')[0]
+
+                    links.append(dict(source=tablename, target = referenced_table))
+
+            nodes.append(dict(name=tablename, type="table", fields = fields))
+
+    # d3 v4 allows individual modules to be specified.  The complete d3 library is included below.
+    response.files.append(URL('admin','static','js/d3.min.js'))
+    response.files.append(URL('admin','static','js/d3_graph.js'))
+    return dict(databases=databases, nodes=nodes, links=links)

applications/examples/static/js/web2py.js

@@ -38,8 +38,12 @@
             if (value > 0) $('#' + id).hide().fadeIn('slow');
             else $('#' + id).show().fadeOut('slow');
         },
-        ajax: function (u, s, t) {
+        ajax: function (u, s, t, options) {
             /*simple ajax function*/
+
+            // set options default value
+            options = typeof options !== 'undefined' ? options : {};
+
             var query = '';
             if (typeof s == 'string') {
                 var d = $(s).serialize();
@@ -59,18 +63,44 @@
                     query = pcs.join('&');
                 }
             }
-            $.ajax({
+
+            // default success action
+            var success_function = function (msg) {
+                if (t) {
+                    if (t == ':eval') eval(msg);
+                    else if (typeof t == 'string') $('#' + t).html(msg);
+                    else t(msg);
+                }
+            };
+
+            // declare success actions as array
+            var success = [success_function];
+
+            // add user success actions
+            if ($.isArray(options.done)){
+                success = $.merge(success, options.done);
+            } else {
+                success.push(options.done);
+            }
+
+            // default jquery ajax options
+            var ajax_options = {
                 type: 'POST',
                 url: u,
                 data: query,
-                success: function (msg) {
-                    if (t) {
-                        if (t == ':eval') eval(msg);
-                        else if (typeof t == 'string') $('#' + t).html(msg);
-                        else t(msg);
-                    }
-                }
-            });
+                success: success
+            };
+
+            //remove custom "done" option if exists
+            delete options.done;
+
+            // merge default ajax options with user custom options
+            for (var attrname in options) {
+                    ajax_options[attrname] = options[attrname];
+            }
+
+            // call ajax function
+            $.ajax(ajax_options);
         },
         ajax_fields: function (target) {
             /*

applications/examples/views/appadmin.html

@@ -233,30 +233,22 @@
 <div class="clear"></div>
 {{pass}}
 
-{{if request.function=='graph_model':}}
+{{if request.function=='d3_graph_model':}}
 <h2>{{=T("Graph Model")}}</h2>
-  {{if not pgv:}}
-    {{=T('pygraphviz library not found')}}
-  {{elif not databases:}}
+  {{if not databases:}}  
     {{=T("No databases in this application")}}
-  {{else:}}	   
-    <div class="btn-group">
-      <a class="btn dropdown-toggle" data-toggle="dropdown" href="#">
-        <i class="icon-download"></i> {{=T('Save model as...')}}
-        <span class="caret"></span>
-      </a>
-      <ul class="dropdown-menu">
-          <li><a href="{{=URL('appadmin', 'bg_graph_model', args=['png'])}}">png</a></li>
-          <li><a href="{{=URL('appadmin', 'bg_graph_model', args=['svg'])}}">svg</a></li>
-          <li><a href="{{=URL('appadmin', 'bg_graph_model', args=['pdf'])}}">pdf</a></li>
-          <li><a href="{{=URL('appadmin', 'bg_graph_model', args=['ps'])}}">ps</a></li>
-          <li><a href="{{=URL('appadmin', 'bg_graph_model', args=['dot'])}}">dot</a></li>
-      </ul>
-    </div>
-    <br />
-    {{=IMG(_src=URL('appadmin', 'bg_graph_model'))}}
+  {{else:}}    
+    <div id="vis"></div>
+      <link rel="stylesheet" href="{{=URL('admin','static','css/d3_graph.css')}}"/>
+      <script>
+        // Define the d3 input data
+        {{from gluon.serializers import json }}
+        var nodes = {{=XML(json(nodes))}};
+        var links = {{=XML(json(links))}};
+        d3_graph();
+      </script>
   {{pass}}
-{{pass}}
+{{pass}}  
 
 {{if request.function == 'manage':}}
 <h2>{{=heading}}</h2>

applications/examples/views/default/download.html

@@ -48,7 +48,7 @@
       </tr>
       <tr>
         <td>
-          <a class="btn btn180 rounded green" href="https://dl.dropbox.com/u/18065445/web2py/web2py_manual_5th.pdf">Manual</a>
+          <a class="btn btn180 rounded green" href="http://mdipierro.github.io/web2py/web2py_manual_5th.pdf">Manual</a>
         </td>
         <td>
           <a class="btn btn180 rounded" href="https://github.com/web2py/web2py/releases">Change Log</a>

applications/examples/views/generic.html

@@ -7,7 +7,7 @@ It is used as default when a view is not provided for your controllers
 """}}
 <h2>{{=' '.join(x.capitalize() for x in request.function.split('_'))}}</h2>
 {{if len(response._vars)==1:}}
-{{=BEAUTIFY(response._vars.values()[0])}}
+{{=BEAUTIFY(response._vars[next(iter(response._vars))])}}
 {{elif len(response._vars)>1:}}
 {{=BEAUTIFY(response._vars)}}
 {{pass}}

applications/examples/views/generic.load

@@ -1 +1 @@
-{{response.headers['web2py-response-flash']=response.flash}}{{if len(response._vars)==1:}}{{=response._vars.values()[0]}}{{else:}}{{=BEAUTIFY(response._vars)}}{{pass}}
+{{response.headers['web2py-response-flash']=response.flash}}{{if len(response._vars)==1:}}{{=response._vars[next(iter(response._vars))]}}{{else:}}{{=BEAUTIFY(response._vars)}}{{pass}}

applications/welcome/controllers/appadmin.py

@@ -12,11 +12,6 @@ import gluon.contenttype
 import gluon.fileutils
 from gluon._compat import iteritems
 
-try:
-    import pygraphviz as pgv
-except ImportError:
-    pgv = None
-
 is_gae = request.env.web2py_runtime_gae or False
 
 # ## critical --- make a copy of the environment
@@ -565,57 +560,6 @@ def table_template(table):
                                           _cellborder=0, _cellspacing=0)
                              ).xml()
 
-
-def bg_graph_model():
-    graph = pgv.AGraph(layout='dot',  directed=True,  strict=False,  rankdir='LR')
-
-    subgraphs = dict()
-    for tablename in db.tables:
-        if hasattr(db[tablename],'_meta_graphmodel'):
-            meta_graphmodel = db[tablename]._meta_graphmodel
-        else:
-            meta_graphmodel = dict(group=request.application, color='#ECECEC')
-
-        group = meta_graphmodel['group'].replace(' ', '')
-        if group not in subgraphs:
-            subgraphs[group] = dict(meta=meta_graphmodel, tables=[])
-        subgraphs[group]['tables'].append(tablename)
-
-        graph.add_node(tablename, name=tablename, shape='plaintext',
-                       label=table_template(tablename))
-
-    for n, key in enumerate(subgraphs.iterkeys()):
-        graph.subgraph(nbunch=subgraphs[key]['tables'],
-                    name='cluster%d' % n,
-                    style='filled',
-                    color=subgraphs[key]['meta']['color'],
-                    label=subgraphs[key]['meta']['group'])
-
-    for tablename in db.tables:
-        for field in db[tablename]:
-            f_type = field.type
-            if isinstance(f_type,str) and (
-                f_type.startswith('reference') or
-                f_type.startswith('list:reference')):
-                referenced_table = f_type.split()[1].split('.')[0]
-                n1 = graph.get_node(tablename)
-                n2 = graph.get_node(referenced_table)
-                graph.add_edge(n1, n2, color="#4C4C4C", label='')
-
-    graph.layout()
-    if not request.args:
-        response.headers['Content-Type'] = 'image/png'
-        return graph.draw(format='png', prog='dot')
-    else:
-        response.headers['Content-Disposition']='attachment;filename=graph.%s'%request.args(0)
-        if request.args(0) == 'dot':
-            return graph.string()
-        else:
-            return graph.draw(format=request.args(0), prog='dot')
-
-def graph_model():
-    return dict(databases=databases, pgv=pgv)
-
 def manage():
     tables = manager_action['tables']
     if isinstance(tables[0], str):
@@ -700,3 +644,51 @@ def hooks():
             ul_t.append(UL([LI(A(f['funcname'], _class="editor_filelink", _href=f['url']if 'url' in f else None, **{'_data-lineno':f['lineno']-1})) for f in op['functions']]))
         ul_main.append(ul_t)
     return ul_main
+
+
+# ##########################################################
+# d3 based model visualizations
+# ###########################################################
+
+def d3_graph_model():
+    """ See https://www.facebook.com/web2py/posts/145613995589010 from Bruno Rocha
+    and also the app_admin bg_graph_model function
+    
+    Create a list of table dicts, called "nodes"
+    """
+    
+    nodes = []
+    links = []
+
+    for database in databases:
+        db = eval_in_global_env(database)
+        for tablename in db.tables:
+            fields = []
+            for field in db[tablename]:
+                f_type = field.type
+                if not isinstance(f_type,str):
+                    disp = ' '
+                elif f_type == 'string':
+                    disp =  field.length
+                elif f_type == 'id':
+                    disp =  "PK"
+                elif f_type.startswith('reference') or \
+                    f_type.startswith('list:reference'):
+                    disp = "FK"
+                else:
+                    disp = ' '
+                fields.append(dict(name= field.name, type=field.type, disp = disp))
+
+                if isinstance(f_type,str) and (
+                    f_type.startswith('reference') or
+                    f_type.startswith('list:reference')):
+                    referenced_table = f_type.split()[1].split('.')[0]
+
+                    links.append(dict(source=tablename, target = referenced_table))
+
+            nodes.append(dict(name=tablename, type="table", fields = fields))
+
+    # d3 v4 allows individual modules to be specified.  The complete d3 library is included below.
+    response.files.append(URL('admin','static','js/d3.min.js'))
+    response.files.append(URL('admin','static','js/d3_graph.js'))
+    return dict(databases=databases, nodes=nodes, links=links)

applications/welcome/routes.example.py

@@ -20,8 +20,8 @@
 # YOU CAN COPY THIS FILE TO ANY APPLICATION'S ROOT DIRECTORY WITHOUT CHANGES!
 # ----------------------------------------------------------------------------------------------------------------------
 
-from fileutils import abspath
-from languages import read_possible_languages
+from gluon.fileutils import abspath
+from gluon.languages import read_possible_languages
 
 possible_languages = read_possible_languages(abspath('applications', app))
 # ----------------------------------------------------------------------------------------------------------------------

applications/welcome/static/js/web2py.js

@@ -12,6 +12,8 @@
         $.error('web2py.js has already been loaded!');
     }
 
+    var FORMDATA_IS_SUPPORTED = typeof(FormData) !== 'undefined';
+
     String.prototype.reverse = function () {
         return this.split('').reverse().join('');
     };
@@ -38,8 +40,12 @@
             if (value > 0) $('#' + id).hide().fadeIn('slow');
             else $('#' + id).show().fadeOut('slow');
         },
-        ajax: function (u, s, t) {
+        ajax: function (u, s, t, options) {
             /*simple ajax function*/
+
+            // set options default value
+            options = typeof options !== 'undefined' ? options : {};
+
             var query = '';
             if (typeof s == 'string') {
                 var d = $(s).serialize();
@@ -59,18 +65,44 @@
                     query = pcs.join('&');
                 }
             }
-            $.ajax({
+
+            // default success action
+            var success_function = function (msg) {
+                if (t) {
+                    if (t == ':eval') eval(msg);
+                    else if (typeof t == 'string') $('#' + t).html(msg);
+                    else t(msg);
+                }
+            };
+
+            // declare success actions as array
+            var success = [success_function];
+
+            // add user success actions
+            if ($.isArray(options.done)){
+                success = $.merge(success, options.done);
+            } else {
+                success.push(options.done);
+            }
+
+            // default jquery ajax options
+            var ajax_options = {
                 type: 'POST',
                 url: u,
                 data: query,
-                success: function (msg) {
-                    if (t) {
-                        if (t == ':eval') eval(msg);
-                        else if (typeof t == 'string') $('#' + t).html(msg);
-                        else t(msg);
-                    }
-                }
-            });
+                success: success
+            };
+
+            //remove custom "done" option if exists
+            delete options.done;
+
+            // merge default ajax options with user custom options
+            for (var attrname in options) {
+                    ajax_options[attrname] = options[attrname];
+            }
+
+            // call ajax function
+            $.ajax(ajax_options);
         },
         ajax_fields: function (target) {
             /*
@@ -233,13 +265,17 @@
                 }
             });
             /* help preventing double form submission for normal form (not LOADed) */
-            $(doc).on('submit', 'form', function () {
-                var submit_button = $(this).find(web2py.formInputClickSelector);
-                web2py.disableElement(submit_button);
+            $(doc).on('submit', 'form', function (e) {
+                var submit_buttons = $(this).find(web2py.formInputClickSelector);
+                submit_buttons.each(function() {
+                    web2py.disableElement($(this));
+                })
                 /* safeguard in case the form doesn't trigger a refresh,
                 see https://github.com/web2py/web2py/issues/1100 */
                 setTimeout(function () {
-                    web2py.enableElement(submit_button);
+                    submit_buttons.each(function() {
+                        web2py.enableElement($(this));
+                    });
                 }, 5000);
             });
             doc.ajaxSuccess(function (e, xhr) {
@@ -290,7 +326,15 @@
                 form.submit(function (e) {
                     web2py.disableElement(form.find(web2py.formInputClickSelector));
                     web2py.hide_flash();
-                    web2py.ajax_page('post', url, form.serialize(), target, form);
+
+                    var formData;
+                    if (FORMDATA_IS_SUPPORTED) {
+                        formData = new FormData(form[0]); // Allows file uploads.
+                    } else {
+                        formData = form.serialize(); // Fallback for older browsers.
+                    }
+                    web2py.ajax_page('post', url, formData, target, form);
+
                     e.preventDefault();
                 });
                 form.on('click', web2py.formInputClickSelector, function (e) {
@@ -309,11 +353,18 @@
             if (web2py.isUndefined(element)) element = $(document);
             /* if target is not there, fill it with something that there isn't in the page*/
             if (web2py.isUndefined(target) || target === '') target = 'w2p_none';
+
+            /* processData and contentType must be set to false when passing a FormData
+               object to jQuery.ajax. */
+            var isFormData = Object.prototype.toString.call(data) === '[object FormData]';
+            var contentType = isFormData ? false : 'application/x-www-form-urlencoded; charset=UTF-8';
             if (web2py.fire(element, 'ajax:before', null, target)) { /*test a usecase, should stop here if returns false */
                 $.ajax({
                     'type': method,
                     'url': action,
                     'data': data,
+                    'processData': !isFormData,
+                    'contentType': contentType,
                     'beforeSend': function (xhr, settings) {
                         xhr.setRequestHeader('web2py-component-location', document.location);
                         xhr.setRequestHeader('web2py-component-element', target);
@@ -669,8 +720,9 @@
             });
         },
         /* Disables form elements:
+        - Does not disable elements with 'data-w2p_disable' attribute
         - Caches element value in 'w2p_enable_with' data store
-        - Replaces element text with value of 'data-disable-with' attribute
+        - Replaces element text with value of 'data-w2p_disable_with' attribute
         - Sets disabled property to true
         */
         disableFormElements: function (form) {
@@ -682,13 +734,15 @@
                 if (!web2py.isUndefined(disable)) {
                     return false;
                 }
-                if (web2py.isUndefined(disable_with)) {
-                    element.data('w2p_disable_with', element[method]());
+                if (!element.is(':file')) { // Altering file input values is not allowed.
+                    if (web2py.isUndefined(disable_with)) {
+                        element.data('w2p_disable_with', element[method]());
+                    }
+                    if (web2py.isUndefined(element.data('w2p_enable_with'))) {
+                        element.data('w2p_enable_with', element[method]());
+                    }
+                    element[method](element.data('w2p_disable_with'));
                 }
-                if (web2py.isUndefined(element.data('w2p_enable_with'))) {
-                    element.data('w2p_enable_with', element[method]());
-                }
-                element[method](element.data('w2p_disable_with'));
                 element.prop('disabled', true);
             });
         },

applications/welcome/views/appadmin.html

@@ -233,30 +233,22 @@
 <div class="clear"></div>
 {{pass}}
 
-{{if request.function=='graph_model':}}
+{{if request.function=='d3_graph_model':}}
 <h2>{{=T("Graph Model")}}</h2>
-  {{if not pgv:}}
-    {{=T('pygraphviz library not found')}}
-  {{elif not databases:}}
+  {{if not databases:}}  
     {{=T("No databases in this application")}}
-  {{else:}}	   
-    <div class="btn-group">
-      <a class="btn dropdown-toggle" data-toggle="dropdown" href="#">
-        <i class="icon-download"></i> {{=T('Save model as...')}}
-        <span class="caret"></span>
-      </a>
-      <ul class="dropdown-menu">
-          <li><a href="{{=URL('appadmin', 'bg_graph_model', args=['png'])}}">png</a></li>
-          <li><a href="{{=URL('appadmin', 'bg_graph_model', args=['svg'])}}">svg</a></li>
-          <li><a href="{{=URL('appadmin', 'bg_graph_model', args=['pdf'])}}">pdf</a></li>
-          <li><a href="{{=URL('appadmin', 'bg_graph_model', args=['ps'])}}">ps</a></li>
-          <li><a href="{{=URL('appadmin', 'bg_graph_model', args=['dot'])}}">dot</a></li>
-      </ul>
-    </div>
-    <br />
-    {{=IMG(_src=URL('appadmin', 'bg_graph_model'))}}
+  {{else:}}    
+    <div id="vis"></div>
+      <link rel="stylesheet" href="{{=URL('admin','static','css/d3_graph.css')}}"/>
+      <script>
+        // Define the d3 input data
+        {{from gluon.serializers import json }}
+        var nodes = {{=XML(json(nodes))}};
+        var links = {{=XML(json(links))}};
+        d3_graph();
+      </script>
   {{pass}}
-{{pass}}
+{{pass}}  
 
 {{if request.function == 'manage':}}
 <h2>{{=heading}}</h2>

applications/welcome/views/generic.html

@@ -7,7 +7,7 @@ It is used as default when a view is not provided for your controllers
 """}}
 <h2>{{=' '.join(x.capitalize() for x in request.function.split('_'))}}</h2>
 {{if len(response._vars)==1:}}
-{{=BEAUTIFY(response._vars.values()[0])}}
+{{=BEAUTIFY(response._vars[next(iter(response._vars))])}}
 {{elif len(response._vars)>1:}}
 {{=BEAUTIFY(response._vars)}}
 {{pass}}

applications/welcome/views/generic.load

@@ -27,4 +27,4 @@ Notice:
 - no need to return a string
 even if the function is called via ajax.
 
-'''}}{{if len(response._vars)==1:}}{{=response._vars.values()[0]}}{{else:}}{{=BEAUTIFY(response._vars)}}{{pass}}
+'''}}{{if len(response._vars)==1:}}{{=response._vars[next(iter(response._vars))]}}{{else:}}{{=BEAUTIFY(response._vars)}}{{pass}}

gluon/_compat.py

@@ -63,7 +63,7 @@ if PY2:
             return None
         if isinstance(obj, (bytes, bytearray, buffer)):
             return bytes(obj)
-        if isinstance(obj, unicode):
+        if hasattr(obj, 'encode'):
             return obj.encode(charset, errors)
         raise TypeError('Expected bytes')
 
@@ -122,7 +122,7 @@ else:
             return None
         if isinstance(obj, (bytes, bytearray, memoryview)):
             return bytes(obj)
-        if isinstance(obj, str):
+        if hasattr(obj, 'encode'):
             return obj.encode(charset, errors)
         raise TypeError('Expected bytes')
 
@@ -151,7 +151,7 @@ def with_metaclass(meta, *bases):
 def to_unicode(obj, charset='utf-8', errors='strict'):
     if obj is None:
         return None
-    if not isinstance(obj, bytes):
+    if not hasattr(obj, 'decode'):
         return text_type(obj)
     return obj.decode(charset, errors)
 

gluon/admin.py

@@ -54,7 +54,8 @@ def app_pack(app, request, raise_ex=False, filenames=None):
 
     """
     try:
-        if filenames is None: app_cleanup(app, request)
+        if filenames is None:
+            app_cleanup(app, request)
         filename = apath('../deposit/web2py.app.%s.w2p' % app, request)
         w2p_pack(filename, apath(app, request), filenames=filenames)
         return filename
@@ -104,7 +105,8 @@ def app_cleanup(app, request):
     if os.path.exists(path):
         for f in os.listdir(path):
             try:
-                if f[:1] != '.': os.unlink(os.path.join(path, f))
+                if f[:1] != '.':
+                    os.unlink(os.path.join(path, f))
             except IOError:
                 r = False
 
@@ -113,7 +115,8 @@ def app_cleanup(app, request):
     if os.path.exists(path):
         for f in os.listdir(path):
             try:
-                if f[:1] != '.': recursive_unlink(os.path.join(path, f))
+                if f[:1] != '.':
+                    recursive_unlink(os.path.join(path, f))
             except (OSError, IOError):
                 r = False
 
@@ -123,7 +126,8 @@ def app_cleanup(app, request):
         CacheOnDisk(folder=path).clear()
         for f in os.listdir(path):
             try:
-                if f[:1] != '.': recursive_unlink(os.path.join(path, f))
+                if f[:1] != '.':
+                    recursive_unlink(os.path.join(path, f))
             except (OSError, IOError):
                 r = False
     return r
@@ -175,10 +179,9 @@ def app_create(app, request, force=False, key=None, info=False):
             return False
     try:
         w2p_unpack('welcome.w2p', path)
-        for subfolder in [
-            'models', 'views', 'controllers', 'databases',
-            'modules', 'cron', 'errors', 'sessions', 'cache',
-            'languages', 'static', 'private', 'uploads']:
+        for subfolder in ['models', 'views', 'controllers', 'databases',
+                          'modules', 'cron', 'errors', 'sessions', 'cache',
+                          'languages', 'static', 'private', 'uploads']:
             subpath = os.path.join(path, subfolder)
             if not os.path.exists(subpath):
                 os.mkdir(subpath)
@@ -368,7 +371,7 @@ def unzip(filename, dir, subfolder=''):
     for name in sorted(zf.namelist()):
         if not name.startswith(subfolder):
             continue
-        #print name[n:]
+        # print name[n:]
         if name.endswith('/'):
             folder = os.path.join(dir, name[n:])
             if not os.path.exists(folder):
@@ -435,6 +438,7 @@ def add_path_first(path):
     if not global_settings.web2py_runtime_gae:
         site.addsitedir(path)
 
+
 def try_mkdir(path):
     if not os.path.exists(path):
         try:
@@ -444,11 +448,12 @@ def try_mkdir(path):
             else:
                 os.mkdir(path)
         except OSError as e:
-            if e.strerror == 'File exists': # In case of race condition.
+            if e.strerror == 'File exists':  # In case of race condition.
                 pass
             else:
                 raise e
 
+
 def create_missing_folders():
     if not global_settings.web2py_runtime_gae:
         for path in ('applications', 'deposit', 'site-packages', 'logs'):

gluon/compileapp.py

@@ -41,11 +41,12 @@ import imp
 import logging
 import types
 from functools import reduce
-logger = logging.getLogger("web2py")
 from gluon import rewrite
 from gluon.custom_import import custom_import_install
 import py_compile
 
+logger = logging.getLogger("web2py")
+
 is_pypy = settings.global_settings.is_pypy
 is_gae = settings.global_settings.web2py_runtime_gae
 is_jython = settings.global_settings.is_jython
@@ -111,7 +112,7 @@ class mybuiltin(object):
     NOTE could simple use a dict and populate it,
     NOTE not sure if this changes things though if monkey patching import.....
     """
-    #__builtins__
+    # __builtins__
     def __getitem__(self, key):
         try:
             return getattr(builtin, key)
@@ -185,7 +186,7 @@ def LOAD(c=None, f='index', args=None, vars=None,
         else:
             statement = "$.web2py.component('%s','%s');" % (url, target)
         attr['_data-w2p_remote'] = url
-        if not target is None:
+        if target is not None:
             return DIV(content, **attr)
 
     else:
@@ -204,14 +205,15 @@ def LOAD(c=None, f='index', args=None, vars=None,
         other_response = Response()
         other_request.env.path_info = '/' + \
             '/'.join([request.application, c, f] +
-                     map(str, other_request.args))
+                     [str(a) for a in other_request.args])
         other_request.env.query_string = \
             vars and URL(vars=vars).split('?')[1] or ''
         other_request.env.http_web2py_component_location = \
             request.env.path_info
         other_request.cid = target
         other_request.env.http_web2py_component_element = target
-        other_request.restful = types.MethodType(request.restful.__func__, other_request) # A bit nasty but needed to use LOAD on action decorates with @request.restful()
+        other_request.restful = types.MethodType(request.restful.__func__, other_request)
+        # A bit nasty but needed to use LOAD on action decorates with @request.restful()
         other_response.view = '%s/%s.%s' % (c, f, other_request.extension)
 
         other_environment = copy.copy(current.globalenv)  # NASTY
@@ -286,7 +288,7 @@ class LoadFactory(object):
             other_response = globals.Response()
             other_request.env.path_info = '/' + \
                 '/'.join([request.application, c, f] +
-                         map(str, other_request.args))
+                         [str(a) for a in other_request.args])
             other_request.env.query_string = \
                 vars and html.URL(vars=vars).split('?')[1] or ''
             other_request.env.http_web2py_component_location = \
@@ -405,7 +407,7 @@ def build_environment(request, response, session, store_current=True):
     """
     Build the environment dictionary into which web2py files are executed.
     """
-    #h,v = html,validators
+    # h,v = html,validators
     environment = dict(_base_environment_)
 
     if not request.env:
@@ -418,7 +420,7 @@ def build_environment(request, response, session, store_current=True):
         r'^%s/%s/\w+\.py$' % (request.controller, request.function)
         ]
 
-    t = environment['T'] = translator(os.path.join(request.folder,'languages'),
+    t = environment['T'] = translator(os.path.join(request.folder, 'languages'),
                                       request.env.http_accept_language)
     c = environment['cache'] = Cache(request)
 
@@ -506,10 +508,12 @@ def compile_models(folder):
         save_pyc(filename)
         os.unlink(filename)
 
+
 def find_exposed_functions(data):
-    data = regex_longcomments.sub('',data)
+    data = regex_longcomments.sub('', data)
     return regex_expose.findall(data)
 
+
 def compile_controllers(folder):
     """
     Compiles all the controllers in the application specified by `folder`
@@ -524,16 +528,19 @@ def compile_controllers(folder):
             command = data + "\nresponse._vars=response._caller(%s)\n" % \
                 function
             filename = pjoin(folder, 'compiled',
-                             'controllers.%s.%s.py' % (fname[:-3],function))
+                             'controllers.%s.%s.py' % (fname[:-3], function))
             write_file(filename, command)
             save_pyc(filename)
             os.unlink(filename)
 
+
 def model_cmp(a, b, sep='.'):
     return cmp(a.count(sep), b.count(sep)) or cmp(a, b)
 
+
 def model_cmp_sep(a, b, sep=os.path.sep):
-    return model_cmp(a,b,sep)
+    return model_cmp(a, b, sep)
+
 
 def run_models_in(environment):
     """
@@ -544,7 +551,7 @@ def run_models_in(environment):
     request = current.request
     folder = request.folder
     c = request.controller
-    #f = environment['request'].function
+    # f = environment['request'].function
     response = current.response
 
     path = pjoin(folder, 'models')
@@ -557,9 +564,11 @@ def run_models_in(environment):
             models = sorted(listdir(path, '^\w+\.py$', 0, sort=False), model_cmp_sep)
     else:
         if compiled:
-            models = sorted(listdir(cpath, '^models[_.][\w.]+\.pyc$', 0), key=lambda f: '{0:03d}'.format(f.count('.')) + f)
+            models = sorted(listdir(cpath, '^models[_.][\w.]+\.pyc$', 0),
+                            key=lambda f: '{0:03d}'.format(f.count('.')) + f)
         else:
-            models = sorted(listdir(path, '^\w+\.py$', 0, sort=False), key=lambda f: '{0:03d}'.format(f.count(os.path.sep)) + f)
+            models = sorted(listdir(path, '^\w+\.py$', 0, sort=False),
+                            key=lambda f: '{0:03d}'.format(f.count(os.path.sep)) + f)
 
     models_to_run = None
     for model in models:
@@ -570,10 +579,10 @@ def run_models_in(environment):
         if models_to_run:
             if compiled:
                 n = len(cpath)+8
-                fname = model[n:-4].replace('.','/')+'.py'
+                fname = model[n:-4].replace('.', '/')+'.py'
             else:
                 n = len(path)+1
-                fname = model[n:].replace(os.path.sep,'/')
+                fname = model[n:].replace(os.path.sep, '/')
             if not regex.search(fname) and c != 'appadmin':
                 continue
             elif compiled:
@@ -583,6 +592,7 @@ def run_models_in(environment):
             ccode = getcfs(model, model, f)
             restricted(ccode, environment, layer=model)
 
+
 def run_controller_in(controller, function, environment):
     """
     Runs the controller.function() (for the app specified by
@@ -596,13 +606,13 @@ def run_controller_in(controller, function, environment):
     badc = 'invalid controller (%s/%s)' % (controller, function)
     badf = 'invalid function (%s/%s)' % (controller, function)
     if os.path.exists(cpath):
-        filename = pjoin(cpath, 'controllers.%s.%s.pyc'
-                         % (controller, function))
-        if not os.path.exists(filename):
+        filename = pjoin(cpath, 'controllers.%s.%s.pyc' % (controller, function))
+        try:
+            ccode = getcfs(filename, filename, lambda: read_pyc(filename))
+        except IOError:
             raise HTTP(404,
                        rewrite.THREAD_LOCAL.routes.error_message % badf,
                        web2py_error=badf)
-        ccode = getcfs(filename, filename, lambda: read_pyc(filename))
     elif function == '_TEST':
         # TESTING: adjust the path to include site packages
         from gluon.settings import global_settings
@@ -623,21 +633,21 @@ def run_controller_in(controller, function, environment):
         code += TEST_CODE
         ccode = compile2(code, filename)
     else:
-        filename = pjoin(folder, 'controllers/%s.py'
-                                 % controller)
-        if not os.path.exists(filename):
+        filename = pjoin(folder, 'controllers/%s.py' % controller)
+        try:
+            code = getcfs(filename, filename, lambda: read_file(filename))
+        except IOError:
             raise HTTP(404,
                        rewrite.THREAD_LOCAL.routes.error_message % badc,
                        web2py_error=badc)
-        code = getcfs(filename, filename, lambda: read_file(filename))
         exposed = find_exposed_functions(code)
-        if not function in exposed:
+        if function not in exposed:
             raise HTTP(404,
                        rewrite.THREAD_LOCAL.routes.error_message % badf,
                        web2py_error=badf)
         code = "%s\nresponse._vars=response._caller(%s)" % (code, function)
         layer = "%s:%s" % (filename, function)
-        ccode = getcfs(layer, filename, lambda: compile2(code, layer))
+        ccode = getcfs(layer, filename, lambda: compile2(code, filename))
 
     restricted(ccode, environment, layer=filename)
     response = environment["response"]
@@ -666,8 +676,9 @@ def run_view_in(environment):
     badv = 'invalid view (%s)' % view
     patterns = response.get('generic_patterns')
     layer = None
+    scode = None
     if patterns:
-        regex = re_compile('|'.join(map(fnmatch.translate, patterns)))
+        regex = re_compile('|'.join(fnmatch.translate(p) for p in patterns))
         short_action = '%(controller)s/%(function)s.%(extension)s' % request
         allow_generic = regex.search(short_action)
     else:
@@ -678,7 +689,7 @@ def run_view_in(environment):
         layer = 'file stream'
     else:
         filename = pjoin(folder, 'views', view)
-        if os.path.exists(cpath): # compiled views
+        if os.path.exists(cpath):  # compiled views
             x = view.replace('/', '.')
             files = ['views.%s.pyc' % x]
             is_compiled = os.path.exists(pjoin(cpath, files[0]))
@@ -698,23 +709,27 @@ def run_view_in(environment):
                         ccode = getcfs(compiled, compiled, lambda: read_pyc(compiled))
                         layer = compiled
                         break
-        if not os.path.exists(filename) and allow_generic:
-            view = 'generic.' + request.extension
-            filename = pjoin(folder, 'views', view)
-        if not os.path.exists(filename):
-            raise HTTP(404,
-                       rewrite.THREAD_LOCAL.routes.error_message % badv,
-                       web2py_error=badv)
-        layer = filename
-        # Compile the template
-        ccode = parse_template(view,
-                               pjoin(folder, 'views'),
-                               context=environment)
-
-    restricted(ccode, environment, layer=layer)
+        # if the view is not compiled
+        if not layer:
+            if not os.path.exists(filename) and allow_generic:
+                view = 'generic.' + request.extension
+                filename = pjoin(folder, 'views', view)
+            if not os.path.exists(filename):
+                raise HTTP(404,
+                           rewrite.THREAD_LOCAL.routes.error_message % badv,
+                           web2py_error=badv)
+            # Parse template
+            scode = parse_template(view,
+                                   pjoin(folder, 'views'),
+                                   context=environment)
+            # Compile template
+            ccode = compile2(scode, filename)
+            layer = filename
+    restricted(ccode, environment, layer=layer, scode=scode)
     # parse_template saves everything in response body
     return environment['response'].body.getvalue()
 
+
 def remove_compiled_application(folder):
     """
     Deletes the folder `compiled` containing the compiled application.

gluon/contenttype.py

@@ -330,7 +330,7 @@ CONTENT_TYPE = {
     '.lha': 'application/x-lha',
     '.lhs': 'text/x-literate-haskell',
     '.lhz': 'application/x-lhz',
-    '.load' : 'text/html',
+    '.load': 'text/html',
     '.log': 'text/x-log',
     '.lrz': 'application/x-lrzip',
     '.ltx': 'text/x-tex',
@@ -823,7 +823,7 @@ CONTENT_TYPE = {
     '.xsd': 'application/xml',
     '.xsl': 'application/xslt+xml',
     '.xslfo': 'text/x-xslfo',
-    '.xslm' : 'application/vnd.ms-excel.sheet.macroEnabled.12',
+    '.xslm': 'application/vnd.ms-excel.sheet.macroEnabled.12',
     '.xslt': 'application/xslt+xml',
     '.xspf': 'application/xspf+xml',
     '.xul': 'application/vnd.mozilla.xul+xml',
@@ -843,7 +843,7 @@ def contenttype(filename, default='text/plain'):
     """
     Returns the Content-Type string matching extension of the given filename.
     """
-    filename=to_native(filename)
+    filename = to_native(filename)
     i = filename.rfind('.')
     if i >= 0:
         default = CONTENT_TYPE.get(filename[i:].lower(), default)

gluon/contrib/heroku.py

@@ -7,15 +7,13 @@ db = get_db()
 """
 import os
 from gluon import *
-from pydal.adapters import ADAPTERS, PostgreSQLAdapter
-from pydal.helpers.classes import UseDatabaseStoredFile
+from pydal.adapters import adapters, PostgrePsyco
+from pydal.helpers.classes import DatabaseStoredFile
 
-class HerokuPostgresAdapter(UseDatabaseStoredFile,PostgreSQLAdapter):
-    drivers = ('psycopg2',)
+@adapters.register_for('postgres')
+class HerokuPostgresAdapter(DatabaseStoredFile, PostgrePsyco):
     uploads_in_blob = True
 
-ADAPTERS['postgres'] = HerokuPostgresAdapter
-
 def get_db(name = None, pool_size=10):
     if not name:
         names = [n for n in os.environ.keys()

gluon/contrib/imageutils.py

@@ -45,7 +45,7 @@ class RESIZE(object):
                 background = Image.new('RGBA', (self.nx, self.ny), (255, 255, 255, 0))
                 background.paste(
                     img,
-                    ((self.nx - img.size[0]) / 2, (self.ny - img.size[1]) / 2))
+                    ((self.nx - img.size[0]) // 2, (self.ny - img.size[1]) // 2))
                 background.save(s, 'JPEG', quality=self.quality)
             else:
                 img.save(s, 'JPEG', quality=self.quality)

gluon/contrib/login_methods/cas_auth.py

@@ -49,7 +49,8 @@ class CasAuth(object):
                            email=lambda v: v.get('email', None),
                            user_id=lambda v: v['user']),
                  casversion=1,
-                 casusername='cas:user'
+                 casusername='cas:user',
+		 change_password_url=None
                  ):
         self.urlbase = urlbase
         self.cas_login_url = "%s/%s" % (self.urlbase, actions[0])
@@ -64,6 +65,9 @@ class CasAuth(object):
                               #vars=current.request.vars,
                               scheme=True)
 
+	# URL to let users change their password in the IDP system
+        self.cas_change_password_url = change_password_url
+
     def login_url(self, next="/"):
         current.session.token = self._CAS_login()
         return next
@@ -74,6 +78,10 @@ class CasAuth(object):
         self._CAS_logout()
         return next
 
+    def change_password_url(self, next="/"):
+        self._CAS_change_password()
+        return next
+
     def get_user(self):
         user = current.session.token
         if user:
@@ -135,3 +143,6 @@ class CasAuth(object):
         redirects to the CAS logout page
         """
         redirect("%s?service=%s" % (self.cas_logout_url, self.cas_my_url))
+
+    def _CAS_change_password(self):
+        redirect(self.cas_change_password_url)

gluon/contrib/login_methods/saml2_auth.py

@@ -145,10 +145,16 @@ class Saml2Auth(object):
             username=lambda v:v['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/upn'][0],
             email=lambda v:v['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/upn'][0],
             user_id=lambda v:v['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/upn'][0],
-            )):
+            ), logout_url=None, change_password_url=None):
         self.config_file = config_file
         self.maps = maps
 
+	# URL for redirecting users to when they sign out
+        self.saml_logout_url = logout_url
+
+        # URL to let users change their password in the IDP system
+        self.saml_change_password_url = change_password_url
+
     def login_url(self, next="/"):
         d = saml2_handler(current.session, current.request)
         if 'url' in d:
@@ -170,6 +176,12 @@ class Saml2Auth(object):
 
     def logout_url(self, next="/"):
         current.session.saml2_info = None
+        current.session.auth = None
+        self._SAML_logout()
+        return next
+
+    def change_password_url(self, next="/"):
+        self._SAML_change_password()
         return next
 
     def get_user(self):        
@@ -180,3 +192,13 @@ class Saml2Auth(object):
                 d[key] = self.maps[key](user)
             return d
         return None
+
+    def _SAML_logout(self):
+        """
+        exposed SAML.logout()
+        redirects to the SAML logout page
+        """
+        redirect(self.saml_logout_url)
+
+    def _SAML_change_password(self):
+        redirect(self.saml_change_password_url)

gluon/contrib/markmin/markmin2html.py

@@ -5,12 +5,19 @@
 # license MIT/BSD/GPL
 from __future__ import print_function
 import re
+import sys
 import urllib
-from gluon._compat import maketrans, urllib_quote, unicodeT, to_bytes, to_native, xrange
-from gluon.utils import local_html_escape as escape
-from ast import parse as ast_parse
 import ast
 
+PY2 = sys.version_info[0] == 2
+
+if PY2:
+    from urllib import quote as urllib_quote
+    from string import maketrans
+else:
+    from urllib.parse import quote as urllib_quote
+    maketrans = str.maketrans
+
 
 """
 TODO: next version should use MathJax
@@ -544,7 +551,7 @@ regex_code = re.compile(
     '(' + META + '|' + DISABLED_META + r'|````)|(``(?P<t>.+?)``(?::(?P<c>[a-zA-Z][_a-zA-Z\-\d]*)(?:\[(?P<p>[^\]]*)\])?)?)',
     re.S)
 regex_strong = re.compile(r'\*\*(?P<t>[^\s*]+( +[^\s*]+)*)\*\*')
-regex_del = re.compile(r'~~(?P<t>[^\s*]+( +[^\s*]+)*)~~')
+regex_del = re.compile(r'~~(?P<t>[^\s~]+( +[^\s~]+)*)~~')
 regex_em = re.compile(r"''(?P<t>([^\s']| |'(?!'))+)''")
 regex_num = re.compile(r"^\s*[+-]?((\d+(\.\d*)?)|\.\d+)([eE][+-]?[0-9]+)?\s*$")
 regex_list = re.compile('^(?:(?:(#{1,6})|(?:(\.+|\++|\-+)(\.)?))\s*)?(.*)$')
@@ -564,6 +571,29 @@ ttab_in = maketrans("'`:*~\\[]{}@$+-.#\n", '\x0b\x0c\x0e\x0f\x10\x11\x12\x13\x14
 ttab_out = maketrans('\x0b\x0c\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x05', "'`:*~\\[]{}@$+-.#\n")
 regex_quote = re.compile('(?P<name>\w+?)\s*\=\s*')
 
+def local_html_escape(data, quote=False):
+    """
+    Works with bytes.
+    Replace special characters "&", "<" and ">" to HTML-safe sequences.
+    If the optional flag quote is true (the default), the quotation mark
+    characters, both double quote (") and single quote (') characters are also
+    translated.
+    """
+    if PY2:
+        import cgi
+        data = cgi.escape(data, quote)
+        return data.replace("'", "&#x27;") if quote else data
+    else:
+        import html
+        if isinstance(data, str):
+            return html.escape(data, quote=quote)
+        data = data.replace(b"&", b"&amp;")  # Must be done first!                                                                                           
+        data = data.replace(b"<", b"&lt;")
+        data = data.replace(b">", b"&gt;")
+        if quote:
+            data = data.replace(b'"', b"&quot;")
+            data = data.replace(b'\'', b"&#x27;")
+        return data
 
 def make_dict(b):
     return '{%s}' % regex_quote.sub("'\g<name>':", b)
@@ -579,7 +609,7 @@ def safe_eval(node_or_string, env):
     _safe_names = {'None': None, 'True': True, 'False': False}
     _safe_names.update(env)
     if isinstance(node_or_string, basestring):
-        node_or_string = ast_parse(node_or_string, mode='eval')
+        node_or_string = ast.parse(node_or_string, mode='eval')
     if isinstance(node_or_string, ast.Expression):
         node_or_string = node_or_string.body
 
@@ -599,14 +629,14 @@ def safe_eval(node_or_string, env):
             if node.id in _safe_names:
                 return _safe_names[node.id]
         elif isinstance(node, ast.BinOp) and \
-                isinstance(node.op, (Add, Sub)) and \
-                isinstance(node.right, Num) and \
+                isinstance(node.op, (ast.Add, ast.Sub)) and \
+                isinstance(node.right, ast.Num) and \
                 isinstance(node.right.n, complex) and \
-                isinstance(node.left, Num) and \
+                isinstance(node.left, ast.Num) and \
                 isinstance(node.left.n, (int, long, float)):
             left = node.left.n
             right = node.right.n
-            if isinstance(node.op, Add):
+            if isinstance(node.op, ast.Add):
                 return left + right
             else:
                 return left - right
@@ -765,7 +795,7 @@ def render(text,
     '<table><tbody><tr class="first"><td>a</td><td>b</td></tr><tr class="even"><td>c</td><td>d</td></tr></tbody></table>'
 
     >>> render("----\\nhello world\\n----\\n")
-    '<blockquote>hello world</blockquote>'
+    '<blockquote><p>hello world</p></blockquote>'
 
     >>> render('[[myanchor]]')
     '<p><span class="anchor" id="markmin_myanchor"></span></p>'
@@ -946,7 +976,8 @@ def render(text,
     if protolinks == "default":
         protolinks = protolinks_simple
     pp = '\n' if pretty_print else ''
-    text = to_native(text)
+    text = text if text is None or isinstance(text, str) else text.decode('utf8', 'strict')
+
     if not (isinstance(text, str)):
         text = str(text or '')
     text = regex_backslash.sub(lambda m: m.group(1).translate(ttab_in), text)
@@ -994,7 +1025,7 @@ def render(text,
         return LINK
 
     text = regex_link.sub(mark_link, text)
-    text = escape(text)
+    text = local_html_escape(text)
 
     if protolinks:
         text = regex_proto.sub(lambda m: protolinks(*m.group('p', 'k')), text)
@@ -1035,7 +1066,7 @@ def render(text,
             if pend and mtag == '.':  # paragraph in a list:
                 out.append(etags.pop())
                 ltags.pop()
-            for i in xrange(lent - lev):
+            for i in range(lent - lev):
                 out.append('<' + tag + '>' + pp)
                 etags.append('</' + tag + '>' + pp)
                 lev += 1
@@ -1044,7 +1075,7 @@ def render(text,
         elif lent == lev:
             if tlev[-1] != tag:
                 # type of list is changed (ul<=>ol):
-                for i in xrange(ltags.count(lent)):
+                for i in range(ltags.count(lent)):
                     ltags.pop()
                     out.append(etags.pop())
                 tlev[-1] = tag
@@ -1209,7 +1240,7 @@ def render(text,
                 s = '<blockquote%s%s>%s</blockquote>%s' \
                     % (t_cls,
                        t_id,
-                       '\n'.join(strings[bq_begin:lineno]), pp)
+                       render('\n'.join(strings[bq_begin:lineno])), pp)
                 mtag = 'q'
         else:
             s = '<hr />'
@@ -1322,10 +1353,10 @@ def render(text,
         t, a, k, p, w = m.group('t', 'a', 'k', 'p', 'w')
         if not k:
             return m.group(0)
-        k = escape(k)
+        k = local_html_escape(k)
         t = t or ''
         style = 'width:%s' % w if w else ''
-        title = ' title="%s"' % escape(a).replace(META, DISABLED_META) if a else ''
+        title = ' title="%s"' % local_html_escape(a).replace(META, DISABLED_META) if a else ''
         p_begin = p_end = ''
         if p == 'center':
             p_begin = '<p style="text-align:center">'
@@ -1349,7 +1380,7 @@ def render(text,
                        autolinks, protolinks, class_prefix, id_prefix, pretty_print)
             return '<%(p)s controls="controls"%(title)s%(style)s><source src="%(k)s" />%(t)s</%(p)s>' \
                    % dict(p=p, title=title, style=style, k=k, t=t)
-        alt = ' alt="%s"' % escape(t).replace(META, DISABLED_META) if t else ''
+        alt = ' alt="%s"' % local_html_escape(t).replace(META, DISABLED_META) if t else ''
         return '%(begin)s<img src="%(k)s"%(alt)s%(title)s%(style)s />%(end)s' \
                % dict(begin=p_begin, k=k, alt=alt, title=title, style=style, end=p_end)
 
@@ -1358,12 +1389,12 @@ def render(text,
         if not k and not t:
             return m.group(0)
         t = t or ''
-        a = escape(a) if a else ''
+        a = local_html_escape(a) if a else ''
         if k:
             if '#' in k and ':' not in k.split('#')[0]:
                 # wikipage, not external url
                 k = k.replace('#', '#' + id_prefix)
-            k = escape(k)
+            k = local_html_escape(k)
             title = ' title="%s"' % a.replace(META, DISABLED_META) if a else ''
             target = ' target="_blank"' if p == 'popup' else ''
             t = render(t, {}, {}, 'br', URL, environment, latex, None,
@@ -1373,7 +1404,7 @@ def render(text,
         if t == 'NEWLINE' and not a:
             return '<br />' + pp
         return '<span class="anchor" id="%s">%s</span>' % (
-            escape(id_prefix + t),
+            local_html_escape(id_prefix + t),
             render(a, {}, {}, 'br', URL,
                    environment, latex, autolinks,
                    protolinks, class_prefix,
@@ -1399,7 +1430,7 @@ def render(text,
     def expand_meta(m):
         code, b, p, s = segments.pop(0)
         if code is None or m.group() == DISABLED_META:
-            return escape(s)
+            return local_html_escape(s)
         if b in extra:
             if code[:1] == '\n':
                 code = code[1:]
@@ -1411,7 +1442,7 @@ def render(text,
                 return str(extra[b](code))
         elif b == 'cite':
             return '[' + ','.join('<a href="#%s" class="%s">%s</a>' %
-                                  (id_prefix + d, b, d) for d in escape(code).split(',')) + ']'
+                                  (id_prefix + d, b, d) for d in local_html_escape(code).split(',')) + ']'
         elif b == 'latex':
             return LATEX % urllib_quote(code)
         elif b in html_colors:
@@ -1426,12 +1457,12 @@ def render(text,
                    % (fg, bg, render(code, {}, {}, 'br', URL, environment, latex,
                                      autolinks, protolinks, class_prefix, id_prefix, pretty_print))
         cls = ' class="%s%s"' % (class_prefix, b) if b and b != 'id' else ''
-        id = ' id="%s%s"' % (id_prefix, escape(p)) if p else ''
+        id = ' id="%s%s"' % (id_prefix, local_html_escape(p)) if p else ''
         beg = (code[:1] == '\n')
         end = [None, -1][code[-1:] == '\n']
         if beg and end:
-            return '<pre><code%s%s>%s</code></pre>%s' % (cls, id, escape(code[1:-1]), pp)
-        return '<code%s%s>%s</code>' % (cls, id, escape(code[beg:end]))
+            return '<pre><code%s%s>%s</code></pre>%s' % (cls, id, local_html_escape(code[1:-1]), pp)
+        return '<code%s%s>%s</code>' % (cls, id, local_html_escape(code[beg:end]))
 
     text = regex_expand_meta.sub(expand_meta, text)
 

gluon/contrib/memcache/__init__.py

@@ -21,6 +21,9 @@ def MemcacheClient(*a, **b):
 
 class MemcacheClientObj(Client):
 
+    def initialize(self):
+        pass
+
     meta_storage = {}
     max_time_expire = 24*3600
 

gluon/contrib/minify/cssmin.py

@@ -10,8 +10,11 @@ Original author: Zachary Voase
 Modified for inclusion into web2py by: Ross Peoples <ross.peoples@gmail.com>
 """
 
+try:
+    from StringIO import StringIO
+except ImportError:
+    from io import StringIO
 
-from StringIO import StringIO  # The pure-Python StringIO supports unicode.
 import re
 
 

gluon/contrib/minify/minify.py

@@ -8,30 +8,40 @@ Created by: Ross Peoples <ross.peoples@gmail.com>
 Modified by: Massimo Di Pierro <massimo.dipierro@gmail.com>
 """
 
-import cssmin
-import jsmin
+from . import cssmin
+from . import jsmin
 import os
 import hashlib
 import re
+import sys
+PY2 = sys.version_info[0] == 2
 
+if PY2:
+    hashlib_md5 = hashlib.md5
+else:
+    hashlib_md5 = lambda s: hashlib.md5(bytes(s, 'utf8'))
+
+def open_py23(filename, mode):
+    if PY2:
+        f = open(filename, mode + 'b')
+    else:
+        f = open(filename, mode, encoding="utf8")
+    return f
 
 def read_binary_file(filename):
-    f = open(filename, 'rb')
+    f = open_py23(filename, 'r')
     data = f.read()
     f.close()
     return data
 
-
 def write_binary_file(filename, data):
-    f = open(filename, 'wb')
+    f = open_py23(filename, 'w')
     f.write(data)
     f.close()
 
-
 def fix_links(css, static_path):
     return re.sub(r'url\((["\'])\.\./', 'url(\\1' + static_path, css)
 
-
 def minify(files, path_info, folder, optimize_css, optimize_js,
            ignore_concat=[],
            ignore_minify=['/jquery.js', '/anytime.js']):
@@ -109,7 +119,7 @@ def minify(files, path_info, folder, optimize_css, optimize_js,
                     js.append(contents)
             else:
                 js.append(filename)
-    dest_key = hashlib.md5(repr(processed)).hexdigest()
+    dest_key = hashlib_md5(repr(processed)).hexdigest()
     if css and concat_css:
         css = '\n\n'.join(contents for contents in css)
         if not inline_css:

gluon/contrib/simplejsonrpc.py

@@ -105,14 +105,14 @@ class ServerProxy(object):
 
     def __getattr__(self, attr):
         "pseudo method that can be called"
-        return lambda *args: self.call(attr, *args)
+        return lambda *args, **vars: self.call(attr, *args, **vars)
 
-    def call(self, method, *args):
+    def call(self, method, *args, **vars):
         "JSON RPC communication (method invocation)"
 
         # build data sent to the service
         request_id = random.randint(0, sys.maxsize)
-        data = {'id': request_id, 'method': method, 'params': args, }
+        data = {'id': request_id, 'method': method, 'params': args or vars, }
         if self.version:
             data['jsonrpc'] = self.version #mandatory key/value for jsonrpc2 validation else err -32600
         request = json.dumps(data)

gluon/contrib/user_agent_parser.py

@@ -673,3 +673,23 @@ def simple_detect(agent):
     if os_version:
         os = " ".join((os, os_version))
     return os, browser
+
+
+class mobilize(object):  
+    """
+    Decorator for controller functions so they use different views for mobile devices.
+
+    WARNING: If you update httpagentparser make sure to leave mobilize for
+    backwards compatibility.
+    """
+    def __init__(self, func):
+        self.func = func
+    
+    def __call__(self):
+        from gluon import current
+        user_agent = current.request.user_agent()
+        if user_agent.is_mobile:
+            items = current.response.view.split('.')
+            items.insert(-1, 'mobile')
+            current.response.view = '.'.join(items)
+        return self.func()

gluon/contrib/webclient.py

@@ -43,6 +43,7 @@ class WebClient(object):
         self.forms = {}
         self.history = []
         self.cookies = {}
+        self.cookiejar = cookielib.CookieJar()
         self.default_headers = default_headers
         self.sessions = {}
         self.session_regex = session_regex and re.compile(session_regex)
@@ -79,9 +80,8 @@ class WebClient(object):
         cookies = cookies or {}
         headers = headers or {}
 
-        cj = cookielib.CookieJar()
         args = [
-            urllib2.HTTPCookieProcessor(cj),
+            urllib2.HTTPCookieProcessor(self.cookiejar),
             urllib2.HTTPHandler(debuglevel=0)
             ]
         # if required do basic auth

gluon/contrib/websocket_messaging.py

@@ -94,32 +94,10 @@ import optparse
 import time
 import sys
 import gluon.utils
-
-if (sys.version_info[0] == 2):
-    from urllib import urlencode, urlopen
-    def to_bytes(obj, charset='utf-8', errors='strict'):
-        if obj is None:
-            return None
-        if isinstance(obj, (bytes, bytearray, buffer)):
-            return bytes(obj)
-        if isinstance(obj, unicode):
-            return obj.encode(charset, errors)
-        raise TypeError('Expected bytes')
-else:
-    from urllib.request import urlopen
-    from urllib.parse import urlencode
-    def to_bytes(obj, charset='utf-8', errors='strict'):
-        if obj is None:
-            return None
-        if isinstance(obj, (bytes, bytearray, memoryview)):
-            return bytes(obj)
-        if isinstance(obj, str):
-            return obj.encode(charset, errors)
-        raise TypeError('Expected bytes')
+from gluon._compat import to_native, to_bytes, urlencode, urlopen
 
 listeners, names, tokens = {}, {}, {}
 
-
 def websocket_send(url, message, hmac_key=None, group='default'):
     sig = hmac_key and hmac.new(to_bytes(hmac_key), to_bytes(message)).hexdigest() or ''
     params = urlencode(
@@ -138,8 +116,8 @@ class PostHandler(tornado.web.RequestHandler):
         if hmac_key and not 'signature' in self.request.arguments:
             self.send_error(401)
         if 'message' in self.request.arguments:
-            message = self.request.arguments['message'][0]
-            group = self.request.arguments.get('group', ['default'])[0]
+            message = self.request.arguments['message'][0].decode(encoding='UTF-8')
+            group = self.request.arguments.get('group', ['default'])[0].decode(encoding='UTF-8')
             print('%s:MESSAGE to %s:%s' % (time.time(), group, message))
             if hmac_key:
                 signature = self.request.arguments['signature'][0]

gluon/custom_import.py

@@ -75,7 +75,7 @@ def custom_importer(name, globals=None, locals=None, fromlist=None, level=-1):
             try:
                 oname = name if not name.startswith('.') else '.'+name
                 return NATIVE_IMPORTER(oname, globals, locals, fromlist, level)
-            except ImportError:
+            except (ImportError, KeyError):
                 items = current.request.folder.split(os.path.sep)
                 if not items[-1]:
                     items = items[:-1]
@@ -100,7 +100,7 @@ def custom_importer(name, globals=None, locals=None, fromlist=None, level=-1):
             import_tb = sys.exc_info()[2]
             try:
                 return NATIVE_IMPORTER(name, globals, locals, fromlist, level)
-            except ImportError as e3:
+            except (ImportError, KeyError) as e3:
                 raise ImportError(e1, import_tb)  # there an import error in the module
         except Exception as e2:
             raise  # there is an error in the module

gluon/dal.py

@@ -14,6 +14,12 @@ from pydal import DAL as DAL
 from pydal import Field
 from pydal.objects import Row, Rows, Table, Query, Set, Expression
 from pydal import SQLCustomType, geoPoint, geoLine, geoPolygon
+from pydal.migrator import Migrator, InDBMigrator
+from gluon.serializers import custom_json, xml
+from gluon.utils import web2py_uuid
+from gluon import sqlhtml
+from pydal.drivers import DRIVERS
+
 
 def _default_validators(db, field):
     """
@@ -97,14 +103,10 @@ def _default_validators(db, field):
     if (field.notnull or field.unique) and field_type not in excluded_fields:
         requires.insert(0, validators.IS_NOT_EMPTY())
     elif not field.notnull and not field.unique and requires:
-        requires[0] = validators.IS_EMPTY_OR(requires[0], null='' if field.type in ('string', 'text', 'password') else None)
+        requires[0] = \
+            validators.IS_EMPTY_OR(requires[0], null='' if field.type in ('string', 'text', 'password') else None)
     return requires
 
-from gluon.serializers import custom_json, xml
-from gluon.utils import web2py_uuid
-from gluon import sqlhtml
-
-
 DAL.serializers = {'json': custom_json, 'xml': xml}
 DAL.validators_method = _default_validators
 DAL.uuid = lambda x: web2py_uuid()
@@ -115,8 +117,7 @@ DAL.representers = {
 DAL.Field = Field
 DAL.Table = Table
 
-#: add web2py contrib drivers to pyDAL
-from pydal.drivers import DRIVERS
+# add web2py contrib drivers to pyDAL
 if not DRIVERS.get('pymysql'):
     try:
         from .contrib import pymysql

gluon/debug.py

@@ -89,7 +89,7 @@ def communicate(command=None):
 
 # New debugger implementation using dbg and a web UI
 
-import gluon.contrib.dbg as dbg
+import gluon.contrib.dbg as c_dbg
 from threading import RLock
 
 interact_lock = RLock()
@@ -109,11 +109,11 @@ def check_interaction(fn):
     return check_fn
 
 
-class WebDebugger(dbg.Frontend):
+class WebDebugger(c_dbg.Frontend):
     """Qdb web2py interface"""
 
     def __init__(self, pipe, completekey='tab', stdin=None, stdout=None):
-        dbg.Frontend.__init__(self, pipe)
+        c_dbg.Frontend.__init__(self, pipe)
         self.clear_interaction()
 
     def clear_interaction(self):
@@ -128,7 +128,7 @@ class WebDebugger(dbg.Frontend):
         run_lock.acquire()
         try:
             while self.pipe.poll():
-                dbg.Frontend.run(self)
+                c_dbg.Frontend.run(self)
         finally:
             run_lock.release()
 
@@ -149,23 +149,23 @@ class WebDebugger(dbg.Frontend):
 
     @check_interaction
     def do_continue(self):
-        dbg.Frontend.do_continue(self)
+        c_dbg.Frontend.do_continue(self)
 
     @check_interaction
     def do_step(self):
-        dbg.Frontend.do_step(self)
+        c_dbg.Frontend.do_step(self)
 
     @check_interaction
     def do_return(self):
-        dbg.Frontend.do_return(self)
+        c_dbg.Frontend.do_return(self)
 
     @check_interaction
     def do_next(self):
-        dbg.Frontend.do_next(self)
+        c_dbg.Frontend.do_next(self)
 
     @check_interaction
     def do_quit(self):
-        dbg.Frontend.do_quit(self)
+        c_dbg.Frontend.do_quit(self)
 
     def do_exec(self, statement):
         interact_lock.acquire()
@@ -175,18 +175,18 @@ class WebDebugger(dbg.Frontend):
                 # avoid spurious interaction notifications:
                 self.set_burst(2)
                 # execute the statement in the remote debugger:
-                return dbg.Frontend.do_exec(self, statement)
+                return c_dbg.Frontend.do_exec(self, statement)
         finally:
             interact_lock.release()
 
 # create the connection between threads:
 
 parent_queue, child_queue = Queue.Queue(), Queue.Queue()
-front_conn = dbg.QueuePipe("parent", parent_queue, child_queue)
-child_conn = dbg.QueuePipe("child", child_queue, parent_queue)
+front_conn = c_dbg.QueuePipe("parent", parent_queue, child_queue)
+child_conn = c_dbg.QueuePipe("child", child_queue, parent_queue)
 
 web_debugger = WebDebugger(front_conn)                                     # frontend
-dbg_debugger = dbg.Qdb(pipe=child_conn, redirect_stdio=False, skip=None)   # backend
+dbg_debugger = c_dbg.Qdb(pipe=child_conn, redirect_stdio=False, skip=None)   # backend
 dbg = dbg_debugger
 
 # enable getting context (stack, globals/locals) at interaction

gluon/decoder.py

@@ -9,19 +9,19 @@ Based on http://code.activestate.com/recipes/52257/
 
 Licensed under the PSF License
 """
-
+from gluon._compat import to_unicode
 import codecs
 
 # None represents a potentially variable byte. "##" in the XML spec...
 autodetect_dict = {  # bytepattern     : ("name",
                                          (0x00, 0x00, 0xFE, 0xFF): ("ucs4_be"),
-    (0xFF, 0xFE, 0x00, 0x00): ("ucs4_le"),
-    (0xFE, 0xFF, None, None): ("utf_16_be"),
-    (0xFF, 0xFE, None, None): ("utf_16_le"),
-    (0x00, 0x3C, 0x00, 0x3F): ("utf_16_be"),
-    (0x3C, 0x00, 0x3F, 0x00): ("utf_16_le"),
-    (0x3C, 0x3F, 0x78, 0x6D): ("utf_8"),
-    (0x4C, 0x6F, 0xA7, 0x94): ("EBCDIC")
+                                         (0xFF, 0xFE, 0x00, 0x00): ("ucs4_le"),
+                                         (0xFE, 0xFF, None, None): ("utf_16_be"),
+                                         (0xFF, 0xFE, None, None): ("utf_16_le"),
+                                         (0x00, 0x3C, 0x00, 0x3F): ("utf_16_be"),
+                                         (0x3C, 0x00, 0x3F, 0x00): ("utf_16_le"),
+                                         (0x3C, 0x3F, 0x78, 0x6D): ("utf_8"),
+                                         (0x4C, 0x6F, 0xA7, 0x94): ("EBCDIC")
 }
 
 
@@ -36,10 +36,10 @@ def autoDetectXMLEncoding(buffer):
     # buffer at once but otherwise we'd have to decode a character at
     # a time looking for the quote character...that's a pain
 
-    encoding = "utf_8"    # according to the XML spec, this is the default
-                          # this code successively tries to refine the default
-                          # whenever it fails to refine, it falls back to
-                          # the last place encoding was set.
+    encoding = "utf_8"
+    # according to the XML spec, this is the default this code successively tries to refine the default
+    # whenever it fails to refine, it falls back to the last place encoding was set.
+
     if len(buffer) >= 4:
         bytes = (byte1, byte2, byte3, byte4) = tuple(map(ord, buffer[0:4]))
         enc_info = autodetect_dict.get(bytes, None)
@@ -51,8 +51,7 @@ def autoDetectXMLEncoding(buffer):
         enc_info = None
 
     if enc_info:
-        encoding = enc_info  # we've got a guess... these are
-                             #the new defaults
+        encoding = enc_info  # we've got a guess... these are the new defaults
 
         # try to find a more precise encoding using xml declaration
         secret_decoder_ring = codecs.lookup(encoding)[1]
@@ -77,4 +76,4 @@ def autoDetectXMLEncoding(buffer):
 
 def decoder(buffer):
     encoding = autoDetectXMLEncoding(buffer)
-    return buffer.decode(encoding).encode('utf8')
+    return to_unicode(buffer, charset=encoding)

gluon/fileutils.py

@@ -415,10 +415,10 @@ def fix_newlines(path):
 |\r|
 )''')
     for filename in listdir(path, '.*\.(py|html)$', drop=False):
-        rdata = read_file(filename, 'rb')
+        rdata = read_file(filename, 'r')
         wdata = regex.sub('\n', rdata)
         if wdata != rdata:
-            write_file(filename, wdata, 'wb')
+            write_file(filename, wdata, 'w')
 
 
 def copystream(

gluon/globals.py

@@ -13,7 +13,8 @@ Contains the classes for the global used variables:
 - Session
 
 """
-from gluon._compat import pickle, StringIO, copyreg, Cookie, urlparse, PY2, iteritems, to_unicode, to_native, unicodeT, long
+from gluon._compat import pickle, StringIO, copyreg, Cookie, urlparse, PY2, iteritems, to_unicode, to_native, \
+    unicodeT, long, hashlib_md5, urllib_quote
 from gluon.storage import Storage, List
 from gluon.streamer import streamer, stream_file_or_304_or_206, DEFAULT_CHUNK_SIZE
 from gluon.contenttype import contenttype
@@ -30,7 +31,7 @@ from gluon.fileutils import copystream
 import hashlib
 from pydal.contrib import portalocker
 from pickle import Pickler, MARK, DICT, EMPTY_DICT
-#from types import DictionaryType
+# from types import DictionaryType
 import datetime
 import re
 import os
@@ -48,7 +49,7 @@ PAST = 'Sat, 1-Jan-1971 00:00:00'
 FUTURE = 'Tue, 1-Dec-2999 23:59:59'
 
 try:
-    #FIXME PY3
+    # FIXME PY3
     from gluon.contrib.minify import minify
     have_minify = True
 except ImportError:
@@ -79,6 +80,7 @@ template_mapping = {
     'js:inline': js_inline
 }
 
+
 # IMPORTANT:
 # this is required so that pickled dict(s) and class.__dict__
 # are sorted and web2py can detect without ambiguity when a session changes
@@ -89,9 +91,11 @@ class SortingPickler(Pickler):
         self._batch_setitems([(key, obj[key]) for key in sorted(obj)])
 
 if PY2:
-#FIXME PY3
     SortingPickler.dispatch = copy.copy(Pickler.dispatch)
     SortingPickler.dispatch[dict] = SortingPickler.save_dict
+else:
+    SortingPickler.dispatch_table = copyreg.dispatch_table.copy()
+    SortingPickler.dispatch_table[dict] = SortingPickler.save_dict
 
 
 def sorting_dumps(obj, protocol=None):
@@ -119,7 +123,7 @@ def copystream_progress(request, chunk_size=10 ** 5):
         dest = tempfile.NamedTemporaryFile()
     except NotImplementedError:  # and GAE this
         dest = tempfile.TemporaryFile()
-    if not 'X-Progress-ID' in request.get_vars:
+    if 'X-Progress-ID' not in request.get_vars:
         copystream(source, dest, size, chunk_size)
         return dest
     cache_key = 'X-Progress-ID:' + request.get_vars['X-Progress-ID']
@@ -197,7 +201,8 @@ class Request(Storage):
         """Takes the QUERY_STRING and unpacks it to get_vars
         """
         query_string = self.env.get('query_string', '')
-        dget = urlparse.parse_qs(query_string, keep_blank_values=1)  # Ref: https://docs.python.org/2/library/cgi.html#cgi.parse_qs
+        dget = urlparse.parse_qs(query_string, keep_blank_values=1)
+        # Ref: https://docs.python.org/2/library/cgi.html#cgi.parse_qs
         get_vars = self._get_vars = Storage(dget)
         for (key, value) in iteritems(get_vars):
             if isinstance(value, list) and len(value) == 1:
@@ -227,8 +232,7 @@ class Request(Storage):
             body.seek(0)
 
         # parse POST variables on POST, PUT, BOTH only in post_vars
-        if (body and not is_json
-            and env.request_method in ('POST', 'PUT', 'DELETE', 'BOTH')):
+        if body and not is_json and env.request_method in ('POST', 'PUT', 'DELETE', 'BOTH'):
             query_string = env.pop('QUERY_STRING', None)
             dpost = cgi.FieldStorage(fp=body, environ=env, keep_blank_values=1)
             try:
@@ -327,11 +331,16 @@ class Request(Storage):
         user_agent = session._user_agent
         if user_agent:
             return user_agent
-        user_agent = user_agent_parser.detect(self.env.http_user_agent)
+        http_user_agent = self.env.http_user_agent or ''
+        user_agent = user_agent_parser.detect(http_user_agent)
         for key, value in user_agent.items():
             if isinstance(value, dict):
                 user_agent[key] = Storage(value)
-        user_agent = session._user_agent = Storage(user_agent)
+        user_agent = Storage(user_agent)
+        user_agent.is_mobile = 'Mobile' in http_user_agent
+        user_agent.is_tablet = 'Tablet' in http_user_agent
+        session._user_agent = user_agent 
+        
         return user_agent
 
     def requires_https(self):
@@ -349,14 +358,14 @@ class Request(Storage):
             current.session.forget()
             redirect(URL(scheme='https', args=self.args, vars=self.vars))
 
-    def restful(self):
+    def restful(self, ignore_extension=False):
         def wrapper(action, request=self):
             def f(_action=action, *a, **b):
                 request.is_restful = True
                 env = request.env
-                is_json = env.content_type=='application/json'
+                is_json = env.content_type == 'application/json'
                 method = env.request_method
-                if len(request.args) and '.' in request.args[-1]:
+                if not ignore_extension and len(request.args) and '.' in request.args[-1]:
                     request.args[-1], _, request.extension = request.args[-1].rpartition('.')
                     current.response.headers['Content-Type'] = \
                         contenttype('.' + request.extension.lower())
@@ -415,7 +424,6 @@ class Response(Storage):
         if not escape:
             self.body.write(str(data))
         else:
-            # FIXME PY3:
             self.body.write(to_native(xmlescape(data)))
 
     def render(self, *a, **b):
@@ -451,13 +459,13 @@ class Response(Storage):
         for meta in iteritems((self.meta or {})):
             k, v = meta
             if isinstance(v, dict):
-                s += '<meta' + ''.join(' %s="%s"' % (xmlescape(key), to_native(xmlescape(v[key]))) for key in v) +' />\n'
+                s += '<meta' + ''.join(' %s="%s"' % (xmlescape(key),
+                                                     to_native(xmlescape(v[key]))) for key in v) + ' />\n'
             else:
                 s += '<meta name="%s" content="%s" />\n' % (k, to_native(xmlescape(v)))
         self.write(s, escape=False)
 
     def include_files(self, extensions=None):
-
         """
         Includes files (usually in the head).
         Can minify and cache local files
@@ -465,46 +473,67 @@ class Response(Storage):
         response.cache_includes = (cache_method, time_expire).
         Example: (cache.disk, 60) # caches to disk for 1 minute.
         """
+        app = current.request.application
+
+        # We start by building a files list in which adjacent files internal to
+        # the application are placed in a list inside the files list.
+        #
+        # We will only minify and concat adjacent internal files as there's
+        # no way to know if changing the order with which the files are apppended
+        # will break things since the order matters in both CSS and JS and 
+        # internal files may be interleaved with external ones.
         files = []
-        ext_files = []
-        has_js = has_css = False
+        # For the adjacent list we're going to use storage List to both distinguish
+        # from the regular list and so we can add attributes
+        internal = List()  
+        internal.has_js = False
+        internal.has_css = False
+        done = set() # to remove duplicates
         for item in self.files:
-            if isinstance(item, (list, tuple)):
-                ext_files.append(item)
+            if not isinstance(item, list):
+                if item in done:
+                    continue
+                done.add(item)
+            if isinstance(item, (list, tuple)) or not item.startswith('/' + app): # also consider items in other web2py applications to be external
+                if internal:
+                    files.append(internal)
+                    internal = List()
+                    internal.has_js = False
+                    internal.has_css = False
+                files.append(item)
                 continue
             if extensions and not item.rpartition('.')[2] in extensions:
                 continue
-            if item in files:
-                continue
+            internal.append(item)
             if item.endswith('.js'):
-                has_js = True
+                internal.has_js = True
             if item.endswith('.css'):
-                has_css = True
-            files.append(item)
+                internal.has_css = True            
+        if internal:
+            files.append(internal)
 
-        if have_minify and ((self.optimize_css and has_css) or (self.optimize_js and has_js)):
-            # cache for 5 minutes by default
-            key = hashlib.md5(repr(files)).hexdigest()
-
-            cache = self.cache_includes or (current.cache.ram, 60 * 5)
-
-            def call_minify(files=files):
-                return minify.minify(files,
-                                     URL('static', 'temp'),
-                                     current.request.folder,
-                                     self.optimize_css,
-                                     self.optimize_js)
-            if cache:
-                cache_model, time_expire = cache
-                files = cache_model('response.files.minified/' + key,
-                                    call_minify,
-                                    time_expire)
-            else:
-                files = call_minify()
-
-        files.extend(ext_files)
-        s = []
-        for item in files:
+        # We're done we can now minify
+        if have_minify:
+            for i, f in enumerate(files):
+                if isinstance(f, List) and ((self.optimize_css and f.has_css) or (self.optimize_js and f.has_js)):
+                    # cache for 5 minutes by default
+                    key = hashlib_md5(repr(f)).hexdigest()
+                    cache = self.cache_includes or (current.cache.ram, 60 * 5)
+                    def call_minify(files=f):
+                        return List(minify.minify(files,
+                                             URL('static', 'temp'),
+                                             current.request.folder,
+                                             self.optimize_css,
+                                             self.optimize_js))
+                    if cache:
+                        cache_model, time_expire = cache
+                        files[i] = cache_model('response.files.minified/' + key,
+                                            call_minify,
+                                            time_expire)
+                    else:
+                        files[i] = call_minify()
+        
+        def static_map(s, item):
             if isinstance(item, str):
                 f = item.lower().split('?')[0]
                 ext = f.rpartition('.')[2]
@@ -523,6 +552,14 @@ class Response(Storage):
                 tmpl = template_mapping.get(f)
                 if tmpl:
                     s.append(tmpl % item[1])
+
+        s = []
+        for item in files:
+            if isinstance(item, List):
+                for f in item:
+                    static_map(s, f)
+            else:
+                static_map(s, item)
         self.write(''.join(s), escape=False)
 
     def stream(self,
@@ -578,9 +615,9 @@ class Response(Storage):
         if hasattr(stream, 'name'):
             filename = stream.name
 
-        if filename and not 'content-type' in keys:
+        if filename and 'content-type' not in keys:
             headers['Content-Type'] = contenttype(filename)
-        if filename and not 'content-length' in keys:
+        if filename and 'content-length' not in keys:
             try:
                 headers['Content-Length'] = \
                     os.path.getsize(filename)
@@ -638,6 +675,11 @@ class Response(Storage):
         if download_filename is None:
             download_filename = filename
         if attachment:
+            # Browsers still don't have a simple uniform way to have non ascii
+            # characters in the filename so for now we are percent encoding it
+            if isinstance(download_filename, unicodeT):
+                download_filename = download_filename.encode('utf-8')
+            download_filename = urllib_quote(download_filename)
             headers['Content-Disposition'] = \
                 'attachment; filename="%s"' % download_filename.replace('"', '\"')
         return self.stream(stream, chunk_size=chunk_size, request=request)
@@ -1022,7 +1064,7 @@ class Session(Storage):
         if self._forget:
             del rcookies[response.session_id_name]
             return
-        if self.get('httponly_cookies',True):
+        if self.get('httponly_cookies', True):
             scookies['HttpOnly'] = True
         if self._secure:
             scookies['secure'] = True
@@ -1193,7 +1235,7 @@ class Session(Storage):
             if (not response.session_id or
                 not response.session_filename or
                 self._forget
-                or self._unchanged(response)):
+                    or self._unchanged(response)):
                 # self.clear_session_cookies()
                 return False
             else:

gluon/highlight.py

@@ -182,8 +182,8 @@ class Highlighter(object):
         )),
         'PYTHONMultilineString': (python_tokenizer,
                                   (('ENDMULTILINESTRING',
-                                  re.compile(r'.*?("""|\'\'\')',
-                                  re.DOTALL), 'color: darkred'), )),
+                                    re.compile(r'.*?("""|\'\'\')',
+                                               re.DOTALL), 'color: darkred'), )),
         'HTML': (html_tokenizer, (
             ('GOTOPYTHON', re.compile(r'\{\{'), 'color: red'),
             ('COMMENT', re.compile(r'<!--[^>]*-->|<!>'),
@@ -209,7 +209,7 @@ class Highlighter(object):
         mode = self.mode
         while i < len(data):
             for (token, o_re, style) in Highlighter.all_styles[mode][1]:
-                if not token in self.suppress_tokens:
+                if token not in self.suppress_tokens:
                     match = o_re.match(data, i)
                     if match:
                         if style:
@@ -221,7 +221,7 @@ class Highlighter(object):
                             new_mode = \
                                 Highlighter.all_styles[mode][0](self,
                                                                 token, match, style)
-                        if not new_mode is None:
+                        if new_mode is not None:
                             mode = new_mode
                         i += max(1, len(match.group()))
                         break
@@ -241,9 +241,9 @@ class Highlighter(object):
             style = self.styles[token]
         if self.span_style != style:
             if style != 'Keep':
-                if not self.span_style is None:
+                if self.span_style is not None:
                     self.output.append('</span>')
-                if not style is None:
+                if style is not None:
                     self.output.append('<span style="%s">' % style)
                 self.span_style = style
 
@@ -260,7 +260,7 @@ def highlight(
 ):
     styles = styles or {}
     attributes = attributes or {}
-    if not 'CODE' in styles:
+    if 'CODE' not in styles:
         code_style = """
         font-size: 11px;
         font-family: Bitstream Vera Sans Mono,monospace;
@@ -272,7 +272,7 @@ def highlight(
         white-space: pre !important;\n"""
     else:
         code_style = styles['CODE']
-    if not 'LINENUMBERS' in styles:
+    if 'LINENUMBERS' not in styles:
         linenumbers_style = """
         font-size: 11px;
         font-family: Bitstream Vera Sans Mono,monospace;
@@ -283,7 +283,7 @@ def highlight(
         color: #A0A0A0;\n"""
     else:
         linenumbers_style = styles['LINENUMBERS']
-    if not 'LINEHIGHLIGHT' in styles:
+    if 'LINEHIGHLIGHT' not in styles:
         linehighlight_style = "background-color: #EBDDE2;"
     else:
         linehighlight_style = styles['LINEHIGHLIGHT']
@@ -333,8 +333,9 @@ def highlight(
                   == '_' and value])
     if fa:
         fa = ' ' + fa
-    return '<table%s><tr style="vertical-align:top;"><td style="min-width:40px; text-align: right;"><pre style="%s">%s</pre></td><td><pre style="%s">%s</pre></td></tr></table>'\
-        % (fa, linenumbers_style, numbers, code_style, code)
+    return '<table%s><tr style="vertical-align:top;">' \
+           '<td style="min-width:40px; text-align: right;"><pre style="%s">%s</pre></td>' \
+           '<td><pre style="%s">%s</pre></td></tr></table>' % (fa, linenumbers_style, numbers, code_style, code)
 
 
 if __name__ == '__main__':
@@ -342,5 +343,4 @@ if __name__ == '__main__':
     argfp = open(sys.argv[1])
     data = argfp.read()
     argfp.close()
-    print('<html><body>' + highlight(data, sys.argv[2])\
-        + '</body></html>')
+    print('<html><body>' + highlight(data, sys.argv[2]) + '</body></html>')

gluon/html.py

@@ -20,7 +20,8 @@ import urllib
 import base64
 from gluon import sanitizer, decoder
 import itertools
-from gluon._compat import reduce, pickle, copyreg, HTMLParser, name2codepoint, iteritems, unichr, unicodeT, urllib_quote, to_bytes, to_native, to_unicode, basestring, urlencode, implements_bool, text_type
+from gluon._compat import reduce, pickle, copyreg, HTMLParser, name2codepoint, iteritems, unichr, unicodeT, \
+    urllib_quote, to_bytes, to_native, to_unicode, basestring, urlencode, implements_bool, text_type, long
 from gluon.utils import local_html_escape
 import marshal
 
@@ -109,6 +110,7 @@ __all__ = [
 
 DEFAULT_PASSWORD_DISPLAY = '*' * 8
 
+
 def xmlescape(data, quote=True):
     """
     Returns an escaped string of the provided data
@@ -124,10 +126,9 @@ def xmlescape(data, quote=True):
 
     if not(isinstance(data, (text_type, bytes))):
         # i.e., integers
-        data=str(data)
+        data = str(data)
     data = to_bytes(data, 'utf8', 'xmlcharrefreplace')
 
-
     # ... and do the escaping
     data = local_html_escape(data, quote)
     return data
@@ -596,10 +597,10 @@ class XML(XmlComponent):
                 for A, IMG and BlockQuote).
                 The key is the tag; the value is a list of allowed attributes.
         """
-        if sanitize:
-            text = sanitizer.sanitize(text, permitted_tags, allowed_attributes)
         if isinstance(text, unicodeT):
             text = to_native(text.encode('utf8', 'xmlcharrefreplace'))
+        if sanitize:
+            text = sanitizer.sanitize(text, permitted_tags, allowed_attributes)
         elif isinstance(text, bytes):
             text = to_native(text)
         elif not isinstance(text, str):
@@ -671,6 +672,7 @@ def XML_pickle(data):
     return XML_unpickle, (marshal.dumps(str(data)),)
 copyreg.pickle(XML, XML_pickle, XML_unpickle)
 
+
 @implements_bool
 class DIV(XmlComponent):
     """
@@ -998,9 +1000,9 @@ class DIV(XmlComponent):
             if isinstance(c, XmlComponent):
                 s = c.flatten(render)
             elif render:
-                s = render(str(c))
+                s = render(to_native(c))
             else:
-                s = str(c)
+                s = to_native(c)
             text += s
         if render:
             text = render(text, self.tag, self.attributes)
@@ -1281,7 +1283,6 @@ class __TAG__(XmlComponent):
     def __getattr__(self, name):
         if name[-1:] == '_':
             name = name[:-1] + '/'
-        name=to_bytes(name)
         return lambda *a, **b: __tag_div__(name, *a, **b)
 
     def __call__(self, html):
@@ -1310,8 +1311,10 @@ class HTML(DIV):
     tag = b'html'
 
     strict = b'<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">\n'
-    transitional = b'<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">\n'
-    frameset = b'<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Frameset//EN" "http://www.w3.org/TR/html4/frameset.dtd">\n'
+    transitional = \
+        b'<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">\n'
+    frameset = \
+        b'<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Frameset//EN" "http://www.w3.org/TR/html4/frameset.dtd">\n'
     html5 = b'<!DOCTYPE HTML>\n'
 
     def xml(self):
@@ -1862,7 +1865,7 @@ class INPUT(DIV):
                 except:
                     import traceback
                     print(traceback.format_exc())
-                    msg = "Validation error, field:%s %s" % (name,validator)
+                    msg = "Validation error, field:%s %s" % (name, validator)
                     raise Exception(msg)
                 if errors is not None:
                     self.vars[name] = value
@@ -1912,7 +1915,7 @@ class INPUT(DIV):
         name = self.attributes.get('_name', None)
         if name and hasattr(self, 'errors') \
                 and self.errors.get(name, None) \
-                and self['hideerror'] != True:
+                and self['hideerror'] is not True:
             self['_class'] = (self['_class'] and self['_class'] + ' ' or '') + 'invalidinput'
             return DIV.xml(self) + DIV(
                 DIV(
@@ -1980,7 +1983,6 @@ class OPTGROUP(DIV):
 
 
 class SELECT(INPUT):
-
     """
     Examples:
 
@@ -2015,7 +2017,7 @@ class SELECT(INPUT):
         if value is not None:
             if not self['_multiple']:
                 for c in options:  # my patch
-                    if ((value is not None) and (str(c['_value']) == str(value))):
+                    if (value is not None) and (str(c['_value']) == str(value)):
                         c['_selected'] = 'selected'
                     else:
                         c['_selected'] = None
@@ -2025,7 +2027,7 @@ class SELECT(INPUT):
                 else:
                     values = [str(value)]
                 for c in options:  # my patch
-                    if ((value is not None) and (str(c['_value']) in values)):
+                    if (value is not None) and (str(c['_value']) in values):
                         c['_selected'] = 'selected'
                     else:
                         c['_selected'] = None
@@ -2376,22 +2378,21 @@ class FORM(DIV):
 
     def as_json(self, sanitize=True):
         d = self.as_dict(flat=True, sanitize=sanitize)
-        from serializers import json
+        from gluon.serializers import json
         return json(d)
 
     def as_yaml(self, sanitize=True):
         d = self.as_dict(flat=True, sanitize=sanitize)
-        from serializers import yaml
+        from gluon.serializers import yaml
         return yaml(d)
 
     def as_xml(self, sanitize=True):
         d = self.as_dict(flat=True, sanitize=sanitize)
-        from serializers import xml
+        from gluon.serializers import xml
         return xml(d)
 
 
 class BEAUTIFY(DIV):
-
     """
     Turns any list, dictionary, etc into decent looking html.
 
@@ -2430,7 +2431,7 @@ class BEAUTIFY(DIV):
         if level == 0:
             return
         for c in self.components:
-            if hasattr(c, 'value') and not callable(c.value):
+            if hasattr(c, 'value') and not callable(c.value) and not isinstance(c, cgi.FieldStorage):
                 if c.value:
                     components.append(c.value)
             if hasattr(c, 'xml') and callable(c.xml):
@@ -2548,7 +2549,7 @@ class MENU(DIV):
                         li['_class'] = li['_class'] + ' ' + self['li_active']
                     else:
                         li['_class'] = self['li_active']
-                if len(item) <= 4 or item[4] == True:
+                if len(item) <= 4 or item[4] is True:
                     ul.append(li)
         return ul
 
@@ -2562,7 +2563,7 @@ class MENU(DIV):
                 # ex: ('', False, A('title', _href=URL(...), _title="title"))
                 # ex: (A('title', _href=URL(...), _title="title"), False, None)
                 custom_items.append(item)
-            elif len(item) <= 4 or item[4] == True:
+            elif len(item) <= 4 or item[4] is True:
                 select.append(OPTION(CAT(prefix, item[0]),
                                      _value=item[2], _selected=item[1]))
                 if len(item) > 3 and len(item[3]):
@@ -2655,36 +2656,24 @@ class web2pyHTMLParser(HTMLParser):
     """
     obj = web2pyHTMLParser(text) parses and html/xml text into web2py helpers.
     obj.tree contains the root of the tree, and tree can be manipulated
-
-    >>> str(web2pyHTMLParser('hello<div a="b" c=3>wor&lt;ld<span>xxx</span>y<script/>yy</div>zzz').tree)
-    'hello<div a="b" c="3">wor&lt;ld<span>xxx</span>y<script></script>yy</div>zzz'
-    >>> str(web2pyHTMLParser('<div>a<span>b</div>c').tree)
-    '<div>a<span>b</span></div>c'
-    >>> tree = web2pyHTMLParser('hello<div a="b">world</div>').tree
-    >>> tree.element(_a='b')['_c']=5
-    >>> str(tree)
-    'hello<div a="b" c="5">world</div>'
     """
+
     def __init__(self, text, closed=('input', 'link')):
         HTMLParser.__init__(self)
         self.tree = self.parent = TAG['']()
         self.closed = closed
-        self.tags = [x for x in __all__ if isinstance(eval(x), DIV)]
         self.last = None
         self.feed(text)
 
     def handle_starttag(self, tagname, attrs):
-        if tagname.upper() in self.tags:
-            tag = eval(tagname.upper())
-        else:
-            if tagname in self.closed:
-                tagname += '/'
-            tag = TAG[tagname]()
+        if tagname in self.closed:
+            tagname += '/'
+        tag = TAG[tagname]()
         for key, value in attrs:
             tag['_' + key] = value
         tag.parent = self.parent
         self.parent.append(tag)
-        if not tag.tag.endswith(b'/'):
+        if not tag.tag.endswith('/'):
             self.parent = tag
         else:
             self.last = tag.tag[:-1]
@@ -2707,7 +2696,6 @@ class web2pyHTMLParser(HTMLParser):
         self.parent.append(entitydefs[name])
 
     def handle_endtag(self, tagname):
-        tagname = to_bytes(tagname)
         # this deals with unbalanced tags
         if tagname == self.last:
             return
@@ -2717,7 +2705,8 @@ class web2pyHTMLParser(HTMLParser):
                 self.parent = self.parent.parent
             except:
                 raise RuntimeError("unable to balance tag %s" % tagname)
-            if parent_tagname[:len(tagname)] == tagname: break
+            if parent_tagname[:len(tagname)] == tagname:
+                break
 
 
 def markdown_serializer(text, tag=None, attr=None):

gluon/http.py

@@ -11,7 +11,7 @@ HTTP statuses helpers
 """
 
 import re
-from gluon._compat import iteritems
+from gluon._compat import iteritems, unicodeT, to_bytes
 
 __all__ = ['HTTP', 'redirect']
 
@@ -111,22 +111,29 @@ class HTTP(Exception):
             if not body:
                 body = status
             if isinstance(body, (str, bytes, bytearray)):
+                if isinstance(body, unicodeT):
+                    body = to_bytes(body) # This must be done before len
                 headers['Content-Length'] = len(body)
         rheaders = []
         for k, v in iteritems(headers):
             if isinstance(v, list):
                 rheaders += [(k, str(item)) for item in v]
-            elif not v is None:
+            elif v is not None:
                 rheaders.append((k, str(v)))
         responder(status, rheaders)
         if env.get('request_method', '') == 'HEAD':
             return ['']
         elif isinstance(body, (str, bytes, bytearray)):
+            if isinstance(body, unicodeT):
+                body = to_bytes(body)
             return [body]
         elif hasattr(body, '__iter__'):
             return body
         else:
-            return [str(body)]
+            body = str(body)
+            if isinstance(body, unicodeT):
+                body = to_bytes(body)
+            return [body]
 
     @property
     def message(self):
@@ -148,7 +155,7 @@ class HTTP(Exception):
             web2py_error=self.headers.get('web2py_error'))
 
     def __str__(self):
-        "stringify me"
+        """stringify me"""
         return self.message
 
 

gluon/import_all.py

@@ -78,13 +78,9 @@ alert_dependency = ['hashlib', 'uuid']
 # Now we remove the blacklisted modules if we are using the stated
 # python version.
 #
-# List of modules deprecated in Python 2.6 or 2.7 that are in the above set
+# List of modules deprecated in Python 2.7 that are in the above list
 py27_deprecated = ['mhlib', 'multifile', 'mimify', 'sets', 'MimeWriter']  # And ['optparse'] but we need it for now
 
-if python_version >= '2.6':
-    base_modules += ['json', 'multiprocessing']
-    base_modules = list(set(base_modules).difference(set(py26_deprecated)))
-
 if python_version >= '2.7':
     base_modules += ['argparse', 'json', 'multiprocessing']
     base_modules = list(set(base_modules).difference(set(py27_deprecated)))

gluon/languages.py

@@ -18,10 +18,11 @@ import pkgutil
 import logging
 from cgi import escape
 from threading import RLock
-from gluon.utf8 import Utf8
+
 from gluon.utils import local_html_escape
 
 from gluon._compat import copyreg, PY2, maketrans, iterkeys, unicodeT, to_unicode, to_bytes, iteritems, to_native, pjoin
+
 from pydal.contrib.portalocker import read_locked, LockedFile
 
 from gluon.fileutils import listdir
@@ -49,8 +50,10 @@ DEFAULT_CONSTRUCT_PLURAL_FORM = lambda word, plural_id: word
 
 if PY2:
     NUMBERS = (int, long, float)
+    from gluon.utf8 import Utf8
 else:
     NUMBERS = (int, float)
+    Utf8 = str
 
 # pattern to find T(blah blah blah) expressions
 PY_STRING_LITERAL_RE = r'(?<=[^\w]T\()(?P<name>'\

gluon/main.py

@@ -11,7 +11,8 @@ The gluon wsgi application
 """
 from __future__ import print_function
 
-if False: import import_all # DO NOT REMOVE PART OF FREEZE PROCESS
+if False:
+    import import_all  # DO NOT REMOVE PART OF FREEZE PROCESS
 import gc
 
 import os
@@ -26,7 +27,7 @@ import random
 import string
 
 from gluon._compat import Cookie, urllib2
-#from thread import allocate_lock
+# from thread import allocate_lock
 
 from gluon.fileutils import abspath, write_file
 from gluon.settings import global_settings
@@ -67,14 +68,14 @@ import gluon.messageboxhandler
 logging.gluon = gluon
 # so we must restore it! Thanks ozancag
 import locale
-locale.setlocale(locale.LC_CTYPE, "C") # IMPORTANT, web2py requires locale "C"
+locale.setlocale(locale.LC_CTYPE, "C")  # IMPORTANT, web2py requires locale "C"
 
 exists = os.path.exists
 pjoin = os.path.join
 
 try:
     logging.config.fileConfig(abspath("logging.conf"))
-except: # fails on GAE or when logfile is missing
+except:  # fails on GAE or when logfile is missing
     logging.basicConfig()
 logger = logging.getLogger("web2py")
 
@@ -254,6 +255,7 @@ class LazyWSGI(object):
                 return [data]
             for item in middleware_apps:
                 app = item(app)
+
             def caller(app):
                 return app(self.environ, self.start_response)
             return lambda caller=caller, app=app: caller(app)
@@ -294,9 +296,9 @@ def wsgibase(environ, responder):
     response = Response()
     session = Session()
     env = request.env
-    #env.web2py_path = global_settings.applications_parent
+    # env.web2py_path = global_settings.applications_parent
     env.web2py_version = web2py_version
-    #env.update(global_settings)
+    # env.update(global_settings)
     static_file = False
     http_response = None
     try:
@@ -325,7 +327,6 @@ def wsgibase(environ, responder):
                             'Expires'] = 'Thu, 31 Dec 2037 23:59:59 GMT'
                     response.stream(static_file, request=request)
 
-
                 # ##################################################
                 # fill in request items
                 # ##################################################
@@ -356,17 +357,15 @@ def wsgibase(environ, responder):
                 cmd_opts = global_settings.cmd_options
 
                 request.update(
-                    client = client,
-                    folder = abspath('applications', app) + os.sep,
-                    ajax = x_req_with == 'xmlhttprequest',
-                    cid = env.http_web2py_component_element,
-                    is_local = (env.remote_addr in local_hosts and
-                                client == env.remote_addr),
-                    is_shell = False,
-                    is_scheduler = False,
-                    is_https = env.wsgi_url_scheme in HTTPS_SCHEMES or \
-                        request.env.http_x_forwarded_proto in HTTPS_SCHEMES \
-                        or env.https == 'on'
+                    client=client,
+                    folder=abspath('applications', app) + os.sep,
+                    ajax=x_req_with == 'xmlhttprequest',
+                    cid=env.http_web2py_component_element,
+                    is_local=(env.remote_addr in local_hosts and client == env.remote_addr),
+                    is_shell=False,
+                    is_scheduler=False,
+                    is_https=env.wsgi_url_scheme in HTTPS_SCHEMES or
+                             request.env.http_x_forwarded_proto in HTTPS_SCHEMES or env.https == 'on'
                     )
                 request.url = environ['PATH_INFO']
 
@@ -390,7 +389,7 @@ def wsgibase(environ, responder):
                                    % 'invalid request',
                                    web2py_error='invalid application')
                 elif not request.is_local and exists(disabled):
-                    five0three = os.path.join(request.folder,'static','503.html')
+                    five0three = os.path.join(request.folder, 'static', '503.html')
                     if os.path.exists(five0three):
                         raise HTTP(503, file(five0three, 'r').read())
                     else:
@@ -406,7 +405,7 @@ def wsgibase(environ, responder):
                 # get the GET and POST data
                 # ##################################################
 
-                #parse_get_post_vars(request, environ)
+                # parse_get_post_vars(request, environ)
 
                 # ##################################################
                 # expose wsgi hooks for convenience
@@ -625,7 +624,7 @@ def appfactory(wsgiapp=wsgibase,
                 raise BaseException("Can't create dir %s" % profiler_dir)
         filepath = pjoin(profiler_dir, 'wtest')
         try:
-            filehandle = open( filepath, 'w' )
+            filehandle = open(filepath, 'w')
             filehandle.close()
             os.unlink(filepath)
         except IOError:
@@ -746,7 +745,7 @@ class HttpServer(object):
         sock_list = [ip, port]
         if not ssl_certificate or not ssl_private_key:
             logger.info('SSL is off')
-        elif not rocket.ssl:
+        elif not rocket.has_ssl:
             logger.warning('Python "ssl" module unavailable. SSL is OFF')
         elif not exists(ssl_certificate):
             logger.warning('unable to open SSL certificate. SSL is OFF')

gluon/newcron.py

@@ -19,13 +19,14 @@ import sched
 import re
 import datetime
 import platform
-import gluon.fileutils
 from functools import reduce
 try:
     import cPickle as pickle
 except:
     import pickle
 from gluon.settings import global_settings
+from gluon import fileutils
+from gluon._compat import to_bytes
 from pydal.contrib import portalocker
 
 logger = logging.getLogger("web2py.cron")
@@ -116,7 +117,7 @@ class Token(object):
     def __init__(self, path):
         self.path = os.path.join(path, 'cron.master')
         if not os.path.exists(self.path):
-            fileutils.write_file(self.path, '', 'wb')
+            fileutils.write_file(self.path, to_bytes(''), 'wb')
         self.master = None
         self.now = time.time()
 
@@ -139,7 +140,7 @@ class Token(object):
         if portalocker.LOCK_EX is None:
             logger.warning('WEB2PY CRON: Disabled because no file locking')
             return None
-        self.master = open(self.path, 'rb+')
+        self.master = fileutils.open_file(self.path, 'rb+')
         try:
             ret = None
             portalocker.lock(self.master, portalocker.LOCK_EX)
@@ -167,6 +168,7 @@ class Token(object):
         """
         Writes into cron.master the time when cron job was completed
         """
+        ret = self.master.closed
         if not self.master.closed:
             portalocker.lock(self.master, portalocker.LOCK_EX)
             logger.debug('WEB2PY CRON: Releasing cron lock')
@@ -177,6 +179,7 @@ class Token(object):
                 pickle.dump((self.now, time.time()), self.master)
             portalocker.unlock(self.master)
             self.master.close()
+        return ret
 
 
 def rangetolist(s, period='min'):
@@ -222,8 +225,8 @@ def parsecronline(line):
     params = line.strip().split(None, 6)
     if len(params) < 7:
         return None
-    daysofweek = {'sun': 0, 'mon': 1, 'tue': 2, 'wed': 3, 'thu': 4,
-                  'fri': 5, 'sat': 6}
+    daysofweek = {'sun': 0, 'mon': 1, 'tue': 2, 'wed': 3,
+                  'thu': 4, 'fri': 5, 'sat': 6}
     for (s, id) in zip(params[:5], ['min', 'hr', 'dom', 'mon', 'dow']):
         if not s in [None, '*']:
             task[id] = []
@@ -236,7 +239,7 @@ def parsecronline(line):
                 elif val.isdigit() or val == '-1':
                     task[id].append(int(val))
                 elif id == 'dow' and val[:3].lower() in daysofweek:
-                    task[id].append(daysofweek(val[:3].lower()))
+                    task[id].append(daysofweek[val[:3].lower()])
     task['user'] = params[5]
     task['cmd'] = params[6]
     return task

gluon/restricted.py

@@ -10,7 +10,7 @@ Restricted environment to execute application's code
 """
 
 import sys
-from gluon._compat import pickle, ClassType
+from gluon._compat import pickle, ClassType, unicodeT, to_bytes
 import traceback
 import types
 import os
@@ -137,7 +137,10 @@ class RestrictedError(Exception):
         self.environment = environment
         if layer:
             try:
-                self.traceback = traceback.format_exc()
+                try:
+                    self.traceback = traceback.format_exc()
+                except:
+                    self.traceback = traceback.format_exc(limit=1)
             except:
                 self.traceback = 'no traceback because template parsing error'
             try:
@@ -189,10 +192,10 @@ class RestrictedError(Exception):
         # safely show an useful message to the user
         try:
             output = self.output
-            if isinstance(output, unicode):
-                output = output.encode("utf8")
-            elif not isinstance(output, str):
+            if not isinstance(output, str, bytes, bytearray):
                 output = str(output)
+            if isinstance(output, unicodeT):
+                output = to_bytes(output)
         except:
             output = ""
         return output
@@ -202,7 +205,7 @@ def compile2(code, layer):
     return compile(code, layer, 'exec')
 
 
-def restricted(ccode, environment=None, layer='Unknown'):
+def restricted(ccode, environment=None, layer='Unknown', scode=None):
     """
     Runs code in environment and returns the output. If an exception occurs
     in code it raises a RestrictedError containing the traceback. Layer is
@@ -227,7 +230,9 @@ def restricted(ccode, environment=None, layer='Unknown'):
             sys.excepthook(etype, evalue, tb)
         del tb
         output = "%s %s" % (etype, evalue)
-        raise RestrictedError(layer, ccode, output, environment)
+        # Save source code in ticket when available
+        scode = scode if scode else ccode
+        raise RestrictedError(layer, scode, output, environment)
 
 
 def snapshot(info=None, context=5, code=None, environment=None):

gluon/rewrite.py

@@ -206,8 +206,7 @@ def url_out(request, environ, application, controller, function,
     if host is True or (host is None and (scheme or port is not None)):
         host = request.env.http_host
     if not scheme or scheme is True:
-        scheme = request.env.get('wsgi_url_scheme', 'http').lower() \
-            if request else 'http'
+        scheme = request.env.get('wsgi_url_scheme', 'http').lower() if request else 'http'
     if host:
         host_port = host if not port else host.split(':', 1)[0] + ':%s' % port
         url = '%s://%s%s' % (scheme, host_port, url)

gluon/rocket.py

@@ -307,13 +307,13 @@ try:
 except ImportError:
     has_futures = False
 
-    class Future:
+    class Future(object):
         pass
 
-    class ThreadPoolExecutor:
+    class ThreadPoolExecutor(object):
         pass
 
-    class _WorkItem:
+    class _WorkItem(object):
         pass
 
 
@@ -784,8 +784,7 @@ class Rocket(object):
                        the application developer.  Please update your \
                        applications to no longer call rocket.stop(True)"
                 try:
-                    import warnings
-                    raise warnings.DeprecationWarning(msg)
+                    raise DeprecationWarning(msg)
                 except ImportError:
                     raise RuntimeError(msg)
 

gluon/sanitizer.py

@@ -11,7 +11,7 @@ Cross-site scripting (XSS) defense
 """
 
 from gluon._compat import HTMLParser, urlparse, entitydefs, basestring
-from cgi import escape
+from gluon.utils import local_html_escape
 from formatter import AbstractFormatter
 from xml.sax.saxutils import quoteattr
 
@@ -21,7 +21,7 @@ __all__ = ['sanitize']
 def xssescape(text):
     """Gets rid of < and > and & and, for good measure, :"""
 
-    return escape(text, quote=True).replace(':', '&#58;')
+    return local_html_escape(text, quote=True).replace(':', '&#58;')
 
 
 class XssCleaner(HTMLParser):
@@ -145,7 +145,7 @@ class XssCleaner(HTMLParser):
         if url.startswith('#'):
             return True
         else:
-            parsed = urlparse(url)
+            parsed = urlparse.urlparse(url)
             return ((parsed[0] in self.allowed_schemes and '.' in parsed[1]) or
                     (parsed[0] in self.allowed_schemes and '@' in parsed[2]) or
                     (parsed[0] == '' and parsed[2].startswith('/')))

gluon/scheduler.py

@@ -1158,7 +1158,7 @@ class Scheduler(MetaScheduler):
         if not self.db_thread:
             logger.debug('thread building own DAL object')
             self.db_thread = DAL(
-                self.db._uri, folder=self.db._adapter.folder)
+                self.db._uri, folder=self.db._adapter.folder, decode_credentials=True)
             self.define_tables(self.db_thread, migrate=False)
         try:
             db = self.db_thread
@@ -1698,7 +1698,7 @@ def main():
     print('groups for this worker: ' + ', '.join(group_names))
     print('connecting to database in folder: ' + options.db_folder or './')
     print('using URI: ' + options.db_uri)
-    db = DAL(options.db_uri, folder=options.db_folder)
+    db = DAL(options.db_uri, folder=options.db_folder, decode_credentials=True)
     print('instantiating scheduler...')
     scheduler = Scheduler(db=db,
                           worker_name=options.worker_name,

gluon/serializers.py

@@ -119,8 +119,8 @@ def xml(value, encoding='UTF-8', key='document', quote=True):
     return ('<?xml version="1.0" encoding="%s"?>' % encoding) + str(xml_rec(value, key, quote))
 
 
-def json(value, default=custom_json, indent=None):
-    value = json_parser.dumps(value, default=default, sort_keys=True, indent=indent)
+def json(value, default=custom_json, indent=None, sort_keys=False):
+    value = json_parser.dumps(value, default=default, sort_keys=sort_keys, indent=indent)
     # replace JavaScript incompatible spacing
     # http://timelessrepo.com/json-isnt-a-javascript-subset
     # PY3 FIXME

gluon/shell.py

@@ -31,10 +31,16 @@ from gluon.globals import Request, Response, Session
 from gluon.storage import Storage, List
 from gluon.admin import w2p_unpack
 from pydal.base import BaseAdapter
-from gluon._compat import iteritems, ClassType
+from gluon._compat import iteritems, ClassType, PY2
 
 logger = logging.getLogger("web2py")
 
+if not PY2:
+    def execfile(filename, global_vars=None, local_vars=None):
+        with open(filename) as f:
+            code = compile(f.read(), filename, 'exec')
+            exec(code, global_vars, local_vars)
+
 
 def enable_autocomplete_and_history(adir, env):
     try:

gluon/sqlhtml.py

@@ -17,6 +17,7 @@ Holds:
 import datetime
 import urllib
 import re
+import copy
 
 import os
 from gluon._compat import StringIO, unichr, urllib_quote, iteritems, basestring, long, unicodeT, to_native, to_unicode
@@ -29,7 +30,7 @@ from gluon.html import URL, FIELDSET, P, DEFAULT_PASSWORD_DISPLAY
 from pydal.base import DEFAULT
 from pydal.objects import Table, Row, Expression, Field, Set, Rows
 from pydal.adapters.base import CALLABLETYPES
-from pydal.helpers.methods import smart_query, bar_encode,  _repr_ref
+from pydal.helpers.methods import smart_query, bar_encode, _repr_ref, merge_tablemaps
 from pydal.helpers.classes import Reference, SQLCustomType
 from gluon.storage import Storage
 from gluon.utils import md5_hash
@@ -405,7 +406,7 @@ class RadioWidget(OptionsWidget):
         cols = attributes.get('cols', 1)
         totals = len(options)
         mods = totals % cols
-        rows = totals / cols
+        rows = totals // cols
         if mods:
             rows += 1
 
@@ -471,7 +472,7 @@ class CheckboxesWidget(OptionsWidget):
         cols = attributes.get('cols', 1)
         totals = len(options)
         mods = totals % cols
-        rows = totals / cols
+        rows = totals // cols
         if mods:
             rows += 1
 
@@ -658,7 +659,8 @@ class AutocompleteWidget(object):
                  orderby=None, limitby=(0, 10), distinct=False,
                  keyword='_autocomplete_%(tablename)s_%(fieldname)s',
                  min_length=2, help_fields=None, help_string=None,
-                 at_beginning=True, default_var='ac'):
+                 at_beginning=True, default_var='ac', user_signature=True,
+                 hash_vars=False):
 
         self.help_fields = help_fields or []
         self.help_string = help_string
@@ -681,12 +683,14 @@ class AutocompleteWidget(object):
         else:
             self.is_reference = False
         if hasattr(request, 'application'):
-            urlvars = request.vars
+            urlvars = copy.copy(request.vars)
             urlvars[default_var] = 1
-            self.url = URL(args=request.args, vars=urlvars)
-            self.callback()
+            self.url = URL(args=request.args, vars=urlvars,
+                           user_signature=user_signature, hash_vars=hash_vars)
+            self.run_callback = True
         else:
             self.url = request
+            self.run_callback = False
 
     def callback(self):
         if self.keyword in self.request.vars:
@@ -759,6 +763,8 @@ class AutocompleteWidget(object):
                 raise HTTP(200, '')
 
     def __call__(self, field, value, **attributes):
+        if self.run_callback:
+            self.callback()
         default = dict(
             _type='text',
             value=(value is not None and str(value)) or '',
@@ -1916,8 +1922,10 @@ class SQLFORM(FORM):
         if 'table_name' in attributes:
             del attributes['table_name']
 
-        return SQLFORM(DAL(None).define_table(table_name, *fields),
-                       **attributes)
+        # Clone fields, while passing tables straight through
+        fields_with_clones = [f.clone() if isinstance(f, Field) else f for f in fields]
+
+        return SQLFORM(DAL(None).define_table(table_name, *fields_with_clones), **attributes)
 
     @staticmethod
     def build_query(fields, keywords):
@@ -1932,11 +1940,13 @@ class SQLFORM(FORM):
             if settings.global_settings.web2py_runtime_gae:
                 return reduce(lambda a,b: a|b, [field.contains(key) for field in sfields])
             else:
+                if not (sfields and key and key.split()):
+                    return fields[0].table
                 return reduce(lambda a,b:a&b,[
                         reduce(lambda a,b: a|b, [
                                 field.contains(k) for field in sfields]
                                ) for k in key.split()])
-
+                    
             # from https://groups.google.com/forum/#!topic/web2py/hKe6lI25Bv4
             # needs testing...
             #words = key.split(' ') if key else []
@@ -2156,6 +2166,7 @@ class SQLFORM(FORM):
              represent_none=None,
              showblobs=False):
 
+        dbset = None
         formstyle = formstyle or current.response.formstyle
         if isinstance(query, Set):
             query = query.query
@@ -2329,7 +2340,7 @@ class SQLFORM(FORM):
             if not isinstance(left, (list, tuple)):
                 left = [left]
             for join in left:
-                tablenames += db._adapter.tables(join)
+                tablenames = merge_tablemaps(tablenames, db._adapter.tables(join))
         tables = [db[tablename] for tablename in tablenames]
         if fields:
             # add missing tablename to virtual fields
@@ -2341,10 +2352,11 @@ class SQLFORM(FORM):
         else:
             fields = []
             columns = []
-            filter1 = lambda f: isinstance(f, Field) and f.readable and (f.type!='blob' or showblobs)
+            filter1 = lambda f: isinstance(f, Field) and (f.type!='blob' or showblobs)
+            filter2 = lambda f: isinstance(f, Field) and f.readable
             for table in tables:
                 fields += filter(filter1, table)
-                columns += filter(filter1, table)
+                columns += filter(filter2, table)
                 for k, f in iteritems(table):
                     if not k.startswith('_'):
                         if isinstance(f, Field.Virtual) and f.readable:
@@ -2553,6 +2565,7 @@ class SQLFORM(FORM):
                         if isinstance(field, Field.Virtual) and not str(field) in expcolumns:
                             expcolumns.append(str(field))
 
+            expcolumns = ['"%s"' % '"."'.join(f.split('.')) for f in expcolumns]
             if export_type in exportManager and exportManager[export_type]:
                 if keywords:
                     try:
@@ -3032,6 +3045,7 @@ class SQLFORM(FORM):
         res.view_form = view_form
         res.search_form = search_form
         res.rows = rows
+        res.dbset = dbset
         return res
 
     @staticmethod
@@ -3150,7 +3164,9 @@ class SQLFORM(FORM):
                 # if isinstance(linked_tables, dict):
                 #     linked_tables = linked_tables.get(table._tablename, [])
                 if linked_tables is None or referee in linked_tables:
-                    field.represent = lambda id, r=None, referee=referee, rep=field.represent: A(callable(rep) and rep(id) or id, cid=request.cid, _href=url(args=['view', referee, id]))
+                    field.represent = (lambda id, r=None, referee=referee, rep=field.represent:
+                                       A(callable(rep) and rep(id) or id,
+                                         cid=request.cid, _href=url(args=['view', referee, id])))
         except (KeyError, ValueError, TypeError):
             redirect(URL(args=table._tablename))
         if nargs == len(args) + 1:
@@ -3555,7 +3571,9 @@ class ExportClass(object):
                 if not self.rows.db._adapter.REGEX_TABLE_DOT_FIELD.match(col):
                     row.append(record._extra[col])
                 else:
-                    (t, f) = col.split('.')
+                    # The grid code modifies rows.colnames, adding double quotes
+                    # around the table and field names -- so they must be removed here.
+                    (t, f) = [name.strip('"') for name in col.split('.')]
                     field = self.rows.db[t][f]
                     if isinstance(record.get(t, None), (Row, dict)):
                         value = record[t][f]

gluon/tests/__init__.py

@@ -13,6 +13,7 @@ from .test_contribs import *
 from .test_routes import *
 from .test_router import *
 from .test_validators import *
+from .test_authapi import *
 from .test_tools import *
 from .test_utils import *
 from .test_serializers import *
@@ -22,7 +23,8 @@ from .test_appadmin import *
 from .test_web import *
 from .test_sqlhtml import *
 from .test_scheduler import *
+from .test_cron import *
+from .test_is_url import *
 
 if sys.version[:3] == '2.7':
-    from .test_is_url import *
     from .test_old_doctests import *

gluon/tests/test_appadmin.py

@@ -17,6 +17,7 @@ from gluon import fileutils
 from gluon.dal import DAL, Field, Table
 from gluon.http import HTTP
 from gluon.fileutils import open_file
+from gluon.cache import CacheInRam
 
 DEFAULT_URI = os.getenv('DB', 'sqlite:memory')
 
@@ -104,6 +105,21 @@ class TestAppAdmin(unittest.TestCase):
         self._test_index()
         remove_compiled_application(appname_path)
 
+    def test_index_minify(self):
+        # test for gluon/contrib/minify
+        self.env['response'].optimize_css = 'concat|minify'
+        self.env['response'].optimize_js = 'concat|minify'
+        self.env['current'].cache = Storage({'ram':CacheInRam()})
+        appname_path = os.path.join(os.getcwd(), 'applications', 'welcome')
+        self._test_index()
+        file_l = os.listdir(os.path.join(appname_path, 'static', 'temp'))
+        file_l.sort()
+        self.assertTrue(len(file_l) == 2)
+        self.assertEqual(file_l[0][0:10], 'compressed')
+        self.assertEqual(file_l[1][0:10], 'compressed')
+        self.assertEqual(file_l[0][-3:], 'css')
+        self.assertEqual(file_l[1][-2:], 'js')
+
     def test_select(self):
         request = self.env['request']
         request.args = List(['db'])

gluon/tests/test_authapi.py

@@ -0,0 +1,171 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+""" Unit tests for authapi """
+import os
+import unittest
+from gluon.globals import Request, Response, Session
+from gluon.languages import translator
+from gluon.dal import DAL, Field
+from gluon.authapi import AuthAPI
+from gluon.storage import Storage
+from gluon._compat import to_bytes, to_native, add_charset
+
+DEFAULT_URI = os.getenv('DB', 'sqlite:memory')
+
+
+class TestAuthAPI(unittest.TestCase):
+
+    def setUp(self):
+        self.request = Request(env={})
+        self.request.application = 'a'
+        self.request.controller = 'c'
+        self.request.function = 'f'
+        self.request.folder = 'applications/admin'
+        self.response = Response()
+        self.session = Session()
+        T = translator('', 'en')
+        self.session.connect(self.request, self.response)
+        from gluon.globals import current
+        self.current = current
+        self.current.request = self.request
+        self.current.response = self.response
+        self.current.session = self.session
+        self.current.T = T
+        self.db = DAL(DEFAULT_URI, check_reserved=['all'])
+        self.auth = AuthAPI(self.db)
+        self.auth.define_tables(username=True, signature=False)
+        # Create a user
+        self.auth.table_user().validate_and_insert(first_name='Bart',
+                                                   last_name='Simpson',
+                                                   username='bart',
+                                                   email='bart@simpson.com',
+                                                   password='bart_password',
+                                                   registration_key='',
+                                                   registration_id=''
+                                                   )
+        self.db.commit()
+
+    def test_login(self):
+        result = self.auth.login(**{'username': 'bart', 'password': 'bart_password'})
+        self.assertTrue(self.auth.is_logged_in())
+        self.assertTrue(result['user']['email'] == 'bart@simpson.com')
+        self.auth.logout()
+        self.assertFalse(self.auth.is_logged_in())
+        self.auth.settings.username_case_sensitive = False
+        result = self.auth.login(**{'username': 'BarT', 'password': 'bart_password'})
+        self.assertTrue(self.auth.is_logged_in())
+
+    def test_logout(self):
+        self.auth.login(**{'username': 'bart', 'password': 'bart_password'})
+        self.assertTrue(self.auth.is_logged_in())
+        result = self.auth.logout()
+        self.assertTrue(not self.auth.is_logged_in())
+        self.assertTrue(result['user'] is None)
+
+    def test_register(self):
+        self.auth.settings.login_after_registration = True
+        result = self.auth.register(**{
+            'username': 'lisa',
+            'first_name': 'Lisa',
+            'last_name': 'Simpson',
+            'email': 'lisa@simpson.com',
+            'password': 'lisa_password'
+        })
+        self.assertTrue(result['user']['email'] == 'lisa@simpson.com')
+        self.assertTrue(self.auth.is_logged_in())
+        with self.assertRaises(AssertionError):  # Can't register if you're logged in
+            result = self.auth.register(**{
+                'username': 'lisa',
+                'first_name': 'Lisa',
+                'last_name': 'Simpson',
+                'email': 'lisa@simpson.com',
+                'password': 'lisa_password'
+            })
+        self.auth.logout()
+        self.auth.settings.login_after_registration = False
+        result = self.auth.register(**{
+            'username': 'barney',
+            'first_name': 'Barney',
+            'last_name': 'Gumble',
+            'email': 'barney@simpson.com',
+            'password': 'barney_password'
+        })
+        self.assertTrue(result['user']['email'] == 'barney@simpson.com')
+        self.assertFalse(self.auth.is_logged_in())
+        self.auth.settings.login_userfield = 'email'
+        result = self.auth.register(**{
+            'username': 'lisa',
+            'first_name': 'Lisa',
+            'last_name': 'Simpson',
+            'email': 'lisa@simpson.com',
+            'password': 'lisa_password'
+        })
+        self.assertTrue(result['errors']['email'] == self.auth.messages.email_taken)
+        self.assertTrue(result['user'] is None)
+        self.auth.settings.registration_requires_verification = True
+        result = self.auth.register(**{
+            'username': 'homer',
+            'first_name': 'Homer',
+            'last_name': 'Simpson',
+            'email': 'homer@simpson.com',
+            'password': 'homer_password'
+        })
+        self.assertTrue('key' in result['user'])
+
+    def test_profile(self):
+        with self.assertRaises(AssertionError):
+            # We are not logged in
+            self.auth.profile()
+        self.auth.login(**{'username': 'bart', 'password': 'bart_password'})
+        self.assertTrue(self.auth.is_logged_in())
+        result = self.auth.profile(email='bartolo@simpson.com')
+        self.assertTrue(result['user']['email'] == 'bartolo@simpson.com')
+        self.assertTrue(self.auth.table_user()[result['user']['id']].email == 'bartolo@simpson.com')
+
+    def test_change_password(self):
+        with self.assertRaises(AssertionError):
+            # We are not logged in
+            self.auth.change_password()
+        self.auth.login(**{'username': 'bart', 'password': 'bart_password'})
+        self.assertTrue(self.auth.is_logged_in())
+        self.auth.change_password(old_password='bart_password', new_password='1234', new_password2='1234')
+        self.auth.logout()
+        self.assertTrue(not self.auth.is_logged_in())
+        self.auth.login(username='bart', password='1234')
+        self.assertTrue(self.auth.is_logged_in())
+        result = self.auth.change_password(old_password='bart_password', new_password='1234', new_password2='5678')
+        self.assertTrue('new_password2' in result['errors'])
+        result = self.auth.change_password(old_password='bart_password', new_password='1234', new_password2='1234')
+        self.assertTrue('old_password' in result['errors'])
+
+    def test_verify_key(self):
+        self.auth.settings.registration_requires_verification = True
+        result = self.auth.register(**{
+            'username': 'homer',
+            'first_name': 'Homer',
+            'last_name': 'Simpson',
+            'email': 'homer@simpson.com',
+            'password': 'homer_password'
+        })
+        self.assertTrue('key' in result['user'])
+        homer_id = result['user']['id']
+        homers_key = result['user']['key']
+        result = self.auth.verify_key(key=None)
+        self.assertTrue(result['errors'] is not None)
+        result = self.auth.verify_key(key='12345')
+        self.assertTrue(result['errors'] is not None)
+        result = self.auth.verify_key(key=homers_key)
+        self.assertTrue(result['errors'] is None)
+        self.assertEqual(self.auth.table_user()[homer_id].registration_key, '')
+        self.auth.settings.registration_requires_approval = True
+        result = self.auth.register(**{
+            'username': 'lisa',
+            'first_name': 'Lisa',
+            'last_name': 'Simpson',
+            'email': 'lisa@simpson.com',
+            'password': 'lisa_password'
+        })
+        lisa_id = result['user']['id']
+        result = self.auth.verify_key(key=result['user']['key'])
+        self.assertEqual(self.auth.table_user()[lisa_id].registration_key, 'pending')

gluon/tests/test_cron.py

@@ -0,0 +1,24 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+""" Unit tests for cron """
+import unittest, os
+from gluon.newcron import Token, crondance
+
+class TestCron(unittest.TestCase):
+
+
+    def test_Token(self):
+        appname_path = os.path.join(os.getcwd(), 'applications', 'welcome')
+        t = Token(path=appname_path)
+        self.assertNotEqual(t.acquire(), None)
+        self.assertFalse(t.release())
+        self.assertEqual(t.acquire(), None)
+        self.assertTrue(t.release())
+        return
+        
+    def test_crondance(self):
+        #TODO update crondance to return something 
+        crondance(os.getcwd())
+        
+        

gluon/tests/test_fileutils.py

@@ -1,10 +1,11 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
+import os
 import unittest
 import datetime
 
-from gluon.fileutils import parse_version
+from gluon.fileutils import parse_version, fix_newlines
 
 
 class TestFileUtils(unittest.TestCase):
@@ -22,3 +23,6 @@ class TestFileUtils(unittest.TestCase):
         # Semantic Beta
         rtn = parse_version('Version 2.14.1-beta+timestamp.2016.03.21.22.35.26')
         self.assertEqual(rtn, (2, 14, 1, 'beta', datetime.datetime(2016, 3, 21, 22, 35, 26)))
+    
+    def test_fix_newlines(self):
+        fix_newlines(os.path.dirname(os.path.abspath(__file__)))

gluon/tests/test_globals.py

@@ -10,27 +10,76 @@ import re
 import unittest
 
 from gluon.globals import Request, Response, Session
+from gluon.rewrite import regex_url_in
 from gluon import URL
 from gluon._compat import basestring
 
+
 def setup_clean_session():
-        request = Request(env={})
-        request.application = 'a'
-        request.controller = 'c'
-        request.function = 'f'
-        request.folder = 'applications/admin'
-        response = Response()
-        session = Session()
-        session.connect(request, response)
+    request = Request(env={})
+    request.application = 'a'
+    request.controller = 'c'
+    request.function = 'f'
+    request.folder = 'applications/admin'
+    response = Response()
+    session = Session()
+    session.connect(request, response)
+    from gluon.globals import current
+    current.request = request
+    current.response = response
+    current.session = session
+    return current
+
+
+class testRequest(unittest.TestCase):
+
+    def setUp(self):
         from gluon.globals import current
-        current.request = request
-        current.response = response
-        current.session = session
-        return current
+        current.response = Response()
+
+    def test_restful_simple(self):
+        env = {'request_method': 'GET', 'PATH_INFO': '/welcome/default/index/1.pdf'}
+        r = Request(env)
+        regex_url_in(r, env)
+
+        @r.restful()
+        def simple_rest():
+            def GET(*args, **vars):
+                return args[0]
+            return locals()
+
+        self.assertEqual(simple_rest(), '1')
+
+    def test_restful_calls_post(self):
+        env = {'request_method': 'POST', 'PATH_INFO': '/welcome/default/index'}
+        r = Request(env)
+        regex_url_in(r, env)
+
+        @r.restful()
+        def post_rest():
+            def POST(*args, **vars):
+                return 'I posted'
+            return locals()
+
+        self.assertEqual(post_rest(), 'I posted')
+
+    def test_restful_ignore_extension(self):
+        env = {'request_method': 'GET', 'PATH_INFO': '/welcome/default/index/127.0.0.1'}
+        r = Request(env)
+        regex_url_in(r, env)
+
+        @r.restful(ignore_extension=True)
+        def ignore_rest():
+            def GET(*args, **vars):
+                return args[0]
+            return locals()
+
+        self.assertEqual(ignore_rest(), '127.0.0.1')
+
 
 class testResponse(unittest.TestCase):
 
-    #port from python 2.7, needed for 2.5 and 2.6 tests
+    # port from python 2.7, needed for 2.5 and 2.6 tests
     def assertRegexpMatches(self, text, expected_regexp, msg=None):
         """Fail the test unless the text matches the regular expression."""
         if isinstance(expected_regexp, basestring):
@@ -99,8 +148,8 @@ class testResponse(unittest.TestCase):
         response.files.append(URL('a', 'static', 'css/file.css'))
         content = return_includes(response)
         self.assertEqual(content,
-            '<script src="https://code.jquery.com/jquery-1.11.3.min.js?var=0" type="text/javascript"></script>' +
-            '<link href="/a/static/css/file.css" rel="stylesheet" type="text/css" />')
+                         '<script src="https://code.jquery.com/jquery-1.11.3.min.js?var=0" type="text/javascript"></script>' +
+                         '<link href="/a/static/css/file.css" rel="stylesheet" type="text/css" />')
 
         response = Response()
         response.files.append(('js', 'http://maps.google.com/maps/api/js?sensor=false'))
@@ -109,12 +158,11 @@ class testResponse(unittest.TestCase):
         response.files.append(URL('a', 'static', 'css/file.ts'))
         content = return_includes(response)
         self.assertEqual(content,
-            '<script src="https://code.jquery.com/jquery-1.11.3.min.js?var=0" type="text/javascript"></script>' +
-            '<link href="/a/static/css/file.css" rel="stylesheet" type="text/css" />' +
-            '<script src="/a/static/css/file.ts" type="text/typescript"></script>' +
-            '<script src="http://maps.google.com/maps/api/js?sensor=false" type="text/javascript"></script>'
-        )
-
+                         '<script src="http://maps.google.com/maps/api/js?sensor=false" type="text/javascript"></script>' +
+                         '<script src="https://code.jquery.com/jquery-1.11.3.min.js?var=0" type="text/javascript"></script>' +
+                         '<link href="/a/static/css/file.css" rel="stylesheet" type="text/css" />' +
+                         '<script src="/a/static/css/file.ts" type="text/typescript"></script>'
+                         )
 
         response = Response()
         response.files.append(URL('a', 'static', 'css/file.js'))
@@ -122,13 +170,13 @@ class testResponse(unittest.TestCase):
         content = return_includes(response, extensions=['css'])
         self.assertEqual(content, '<link href="/a/static/css/file.css" rel="stylesheet" type="text/css" />')
 
-        #regr test for #628
+        # regr test for #628
         response = Response()
         response.files.append('http://maps.google.com/maps/api/js?sensor=false')
         content = return_includes(response)
         self.assertEqual(content, '')
 
-        #regr test for #628
+        # regr test for #628
         response = Response()
         response.files.append(('js', 'http://maps.google.com/maps/api/js?sensor=false'))
         content = return_includes(response)
@@ -147,7 +195,7 @@ class testResponse(unittest.TestCase):
     def test_cookies(self):
         current = setup_clean_session()
         cookie = str(current.response.cookies)
-        session_key='%s=%s'%(current.response.session_id_name,current.response.session_id)
+        session_key = '%s=%s' % (current.response.session_id_name, current.response.session_id)
         self.assertRegexpMatches(cookie, r'^Set-Cookie: ')
         self.assertTrue(session_key in cookie)
         self.assertTrue('Path=/' in cookie)

gluon/tests/test_html.py

@@ -11,11 +11,13 @@ import unittest
 from gluon.html import A, ASSIGNJS, B, BEAUTIFY, P, BODY, BR, BUTTON, CAT, CENTER, CODE, COL, COLGROUP, DIV, SPAN, URL, verifyURL
 from gluon.html import truncate_string, EM, FIELDSET, FORM, H1, H2, H3, H4, H5, H6, HEAD, HR, HTML, I, IFRAME, IMG, INPUT, EMBED
 from gluon.html import LABEL, LEGEND, LI, LINK, MARKMIN, MENU, META, OBJECT, OL, OPTGROUP, OPTION, PRE, SCRIPT, SELECT, STRONG
-from gluon.html import STYLE, TABLE, TR, TD, TAG, TBODY, THEAD, TEXTAREA, TFOOT, TH, TITLE, TT, UL, XHTML, XML
+from gluon.html import STYLE, TABLE, TR, TD, TAG, TBODY, THEAD, TEXTAREA, TFOOT, TH, TITLE, TT, UL, XHTML, XML, web2pyHTMLParser
 from gluon.storage import Storage
 from gluon.html import XML_pickle, XML_unpickle
 from gluon.html import TAG_pickler, TAG_unpickler
 from gluon._compat import xrange, PY2, to_native
+from gluon.decoder import decoder
+import re
 
 class TestBareHelpers(unittest.TestCase):
 
@@ -155,7 +157,7 @@ class TestBareHelpers(unittest.TestCase):
         self.assertEqual(rtn, True)
 
     # TODO: def test_XmlComponent(self):
-    @unittest.skipIf(not PY2, "Skipping Python 3.x tests for XML.__repr__")
+
     def test_XML(self):
         # sanitization process
         self.assertEqual(XML('<h1>Hello<a data-hello="world">World</a></h1>').xml(),
@@ -168,6 +170,8 @@ class TestBareHelpers(unittest.TestCase):
         # seams that __repr__ is no longer enough
         ##self.assertEqual(XML('1.3'), '1.3')
         self.assertEqual(XML(u'<div>è</div>').xml(), b'<div>\xc3\xa8</div>')
+        # make sure unicode works with sanitize
+        self.assertEqual(XML(u'<div>è</div>', sanitize=True).xml(), b'<div>\xc3\xa8</div>')
         # you can calc len on the class, that equals the xml() and the str()
         ##self.assertEqual(len(XML('1.3')), len('1.3'))
         self.assertEqual(len(XML('1.3').xml()), len('1.3'))
@@ -179,19 +183,18 @@ class TestBareHelpers(unittest.TestCase):
         # you can compare them
         ##self.assertEqual(XML('a') == XML('a'), True)
         # beware that the comparison is made on the XML repr
-        self.assertEqual(XML('<h1>Hello<a data-hello="world">World</a></h1>', sanitize=True),
-                         XML('<h1>HelloWorld</h1>'))
+
+        self.assertEqual(XML('<h1>Hello<a data-hello="world">World</a></h1>', sanitize=True).__repr__(),
+                         XML('<h1>HelloWorld</h1>').__repr__())
         # bug check for the sanitizer for closing no-close tags
-        self.assertEqual(XML('<p>Test</p><br/><p>Test</p><br/>', sanitize=True),
-                         XML('<p>Test</p><br /><p>Test</p><br />'))
+        self.assertEqual(XML('<p>Test</p><br/><p>Test</p><br/>', sanitize=True).xml(),
+                         XML('<p>Test</p><br /><p>Test</p><br />').xml())
         # basic flatten test
         self.assertEqual(XML('<p>Test</p>').flatten(), '<p>Test</p>')
         self.assertEqual(XML('<p>Test</p>').flatten(render=lambda text, tag, attr: text), '<p>Test</p>')
 
-    @unittest.skipIf(not PY2, "Skipping Python 3.x tests for XML_unpickle.__repr__")
     def test_XML_pickle_unpickle(self):
-        # weird test
-        self.assertEqual(XML_unpickle(XML_pickle('data to be pickle')[1][0]), 'data to be pickle')
+        self.assertEqual(str(XML_unpickle(XML_pickle('data to be pickle')[1][0])), 'data to be pickle')
 
     def test_DIV(self):
         # Empty DIV()
@@ -255,6 +258,11 @@ class TestBareHelpers(unittest.TestCase):
         self.assertEqual(DIV('<p>Test</p>', _class="class_test").get('_class'), 'class_test')
         self.assertEqual(DIV(b'a').xml(), b'<div>a</div>')
 
+    def test_decoder(self):
+        tag_html = '<div><span><a id="1-1" u:v="$">hello</a></span><p class="this is a test">world</p></div>'
+        a = decoder(tag_html)
+        self.assertEqual(a, tag_html)
+
     def test_CAT(self):
         # Empty CAT()
         self.assertEqual(CAT().xml(), b'')
@@ -636,8 +644,8 @@ class TestBareHelpers(unittest.TestCase):
         # These 2 crash AppVeyor and Travis with: "ImportError: No YAML serializer available"
         # self.assertEqual(FORM('<>', _a='1', _b='2').as_yaml(),
         #                  "accepted: null\nattributes: {_a: '1', _action: '#', _b: '2', _enctype: multipart/form-data, _method: post}\ncomponents: [<>]\nerrors: {}\nlatest: {}\nparent: null\nvars: {}\n")
-        # self.assertEqual(FORM('<>', _a='1', _b='2').as_xml(),
-        #                  '<?xml version="1.0" encoding="UTF-8"?><document><errors></errors><vars></vars><parent>None</parent><attributes><_enctype>multipart/form-data</_enctype><_action>#</_action><_b>2</_b><_a>1</_a><_method>post</_method></attributes><components><item>&amp;lt;&amp;gt;</item></components><accepted>None</accepted><latest></latest></document>')
+        # TODO check tags content
+        self.assertEqual(len(FORM('<>', _a='1', _b='2').as_xml()), 334)
 
     def test_BEAUTIFY(self):
         #self.assertEqual(BEAUTIFY(['a', 'b', {'hello': 'world'}]).xml(),
@@ -670,13 +678,42 @@ class TestBareHelpers(unittest.TestCase):
 
     # TODO: def test_embed64(self):
 
-    # TODO: def test_web2pyHTMLParser(self):
+    def test_web2pyHTMLParser(self):
+        #tag should not be a byte
+        self.assertEqual(web2pyHTMLParser("<div></div>").tree.components[0].tag, 'div')
+        a = str(web2pyHTMLParser('<div>a<span>b</div>c').tree)
+        self.assertEqual(a, "<div>a<span>b</span></div>c")
+
+        tree = web2pyHTMLParser('hello<div a="b">world</div>').tree
+        tree.element(_a='b')['_c']=5
+        self.assertEqual(str(tree), 'hello<div a="b" c="5">world</div>')
+
+        a = str(web2pyHTMLParser('<div><img class="img"/></div>', closed=['img']).tree)
+        self.assertEqual(a, '<div><img class="img" /></div>')
+
+        #greater-than sign ( > )  --> decimal &#62; --> hexadecimal &#x3E;
+        #Less-than sign    ( < )  --> decimal &#60; --> hexadecimal &#x3C;
+        # test decimal
+        a = str(web2pyHTMLParser('<div>&#60; &#62;</div>').tree)
+        self.assertEqual(a, '<div>&lt; &gt;</div>')
+        # test hexadecimal
+        a = str(web2pyHTMLParser('<div>&#x3C; &#x3E;</div>').tree)
+        self.assertEqual(a, '<div>&lt; &gt;</div>')
+
+    def test_markdown(self):
+        def markdown(text, tag=None, attributes={}):
+            r = {None: re.sub('\s+',' ',text), \
+                 'h1':'#'+text+'\\n\\n', \
+                 'p':text+'\\n'}.get(tag,text)
+            return r
+        a=TAG('<h1>Header</h1><p>this is a     test</p>')
+        ret = a.flatten(markdown)
+        self.assertEqual(ret, '#Header\\n\\nthis is a test\\n')
 
     # TODO: def test_markdown_serializer(self):
 
     # TODO: def test_markmin_serializer(self):
 
-    @unittest.skipIf(not PY2, "Skipping Python 3.x tests for MARKMIN")
     def test_MARKMIN(self):
         # This test pass with python 2.7 but expected to fail under 2.6
         # with self.assertRaises(TypeError) as cm:

gluon/tests/test_is_url.py

@@ -586,81 +586,81 @@ class TestUnicode(unittest.TestCase):
                # disables prepending the scheme in the return value
 
     def testUnicodeToAsciiUrl(self):
-        self.assertEquals(unicode_to_ascii_authority(u'www.Alliancefran\xe7aise.nu'), 'www.xn--alliancefranaise-npb.nu')
-        self.assertEquals(
+        self.assertEqual(unicode_to_ascii_authority(u'www.Alliancefran\xe7aise.nu'), 'www.xn--alliancefranaise-npb.nu')
+        self.assertEqual(
             unicode_to_ascii_authority(u'www.benn.ca'), 'www.benn.ca')
         self.assertRaises(UnicodeError, unicode_to_ascii_authority,
                           u'\u4e2d' * 1000)  # label is too long
 
     def testValidUrls(self):
-        self.assertEquals(self.x(u'www.Alliancefrancaise.nu'), (
+        self.assertEqual(self.x(u'www.Alliancefrancaise.nu'), (
             'http://www.Alliancefrancaise.nu', None))
-        self.assertEquals(self.x(u'www.Alliancefran\xe7aise.nu'), (
+        self.assertEqual(self.x(u'www.Alliancefran\xe7aise.nu'), (
             'http://www.xn--alliancefranaise-npb.nu', None))
-        self.assertEquals(self.x(u'www.Alliancefran\xe7aise.nu:8080'), (
+        self.assertEqual(self.x(u'www.Alliancefran\xe7aise.nu:8080'), (
             'http://www.xn--alliancefranaise-npb.nu:8080', None))
-        self.assertEquals(self.x(u'http://www.Alliancefran\xe7aise.nu'),
+        self.assertEqual(self.x(u'http://www.Alliancefran\xe7aise.nu'),
                           ('http://www.xn--alliancefranaise-npb.nu', None))
-        self.assertEquals(self.x(u'http://www.Alliancefran\xe7aise.nu/parnaise/blue'), ('http://www.xn--alliancefranaise-npb.nu/parnaise/blue', None))
-        self.assertEquals(self.x(u'http://www.Alliancefran\xe7aise.nu/parnaise/blue#fragment'), ('http://www.xn--alliancefranaise-npb.nu/parnaise/blue#fragment', None))
-        self.assertEquals(self.x(u'http://www.Alliancefran\xe7aise.nu/parnaise/blue?query=value#fragment'), ('http://www.xn--alliancefranaise-npb.nu/parnaise/blue?query=value#fragment', None))
-        self.assertEquals(self.x(u'http://www.Alliancefran\xe7aise.nu:8080/parnaise/blue?query=value#fragment'), ('http://www.xn--alliancefranaise-npb.nu:8080/parnaise/blue?query=value#fragment', None))
-        self.assertEquals(self.x(u'www.Alliancefran\xe7aise.nu/parnaise/blue?query=value#fragment'), ('http://www.xn--alliancefranaise-npb.nu/parnaise/blue?query=value#fragment', None))
-        self.assertEquals(self.x(
+        self.assertEqual(self.x(u'http://www.Alliancefran\xe7aise.nu/parnaise/blue'), ('http://www.xn--alliancefranaise-npb.nu/parnaise/blue', None))
+        self.assertEqual(self.x(u'http://www.Alliancefran\xe7aise.nu/parnaise/blue#fragment'), ('http://www.xn--alliancefranaise-npb.nu/parnaise/blue#fragment', None))
+        self.assertEqual(self.x(u'http://www.Alliancefran\xe7aise.nu/parnaise/blue?query=value#fragment'), ('http://www.xn--alliancefranaise-npb.nu/parnaise/blue?query=value#fragment', None))
+        self.assertEqual(self.x(u'http://www.Alliancefran\xe7aise.nu:8080/parnaise/blue?query=value#fragment'), ('http://www.xn--alliancefranaise-npb.nu:8080/parnaise/blue?query=value#fragment', None))
+        self.assertEqual(self.x(u'www.Alliancefran\xe7aise.nu/parnaise/blue?query=value#fragment'), ('http://www.xn--alliancefranaise-npb.nu/parnaise/blue?query=value#fragment', None))
+        self.assertEqual(self.x(
             u'http://\u4e2d\u4fd4.com'), ('http://xn--fiq13b.com', None))
-        self.assertEquals(self.x(u'http://\u4e2d\u4fd4.com/\u4e86'),
+        self.assertEqual(self.x(u'http://\u4e2d\u4fd4.com/\u4e86'),
                           ('http://xn--fiq13b.com/%4e%86', None))
-        self.assertEquals(self.x(u'http://\u4e2d\u4fd4.com/\u4e86?query=\u4e86'), ('http://xn--fiq13b.com/%4e%86?query=%4e%86', None))
-        self.assertEquals(self.x(u'http://\u4e2d\u4fd4.com/\u4e86?query=\u4e86#fragment'), ('http://xn--fiq13b.com/%4e%86?query=%4e%86#fragment', None))
-        self.assertEquals(self.x(u'http://\u4e2d\u4fd4.com?query=\u4e86#fragment'), ('http://xn--fiq13b.com?query=%4e%86#fragment', None))
-        self.assertEquals(
+        self.assertEqual(self.x(u'http://\u4e2d\u4fd4.com/\u4e86?query=\u4e86'), ('http://xn--fiq13b.com/%4e%86?query=%4e%86', None))
+        self.assertEqual(self.x(u'http://\u4e2d\u4fd4.com/\u4e86?query=\u4e86#fragment'), ('http://xn--fiq13b.com/%4e%86?query=%4e%86#fragment', None))
+        self.assertEqual(self.x(u'http://\u4e2d\u4fd4.com?query=\u4e86#fragment'), ('http://xn--fiq13b.com?query=%4e%86#fragment', None))
+        self.assertEqual(
             self.x(u'http://B\xfccher.ch'), ('http://xn--bcher-kva.ch', None))
-        self.assertEquals(self.x(u'http://\xe4\xf6\xfc\xdf.com'), (
+        self.assertEqual(self.x(u'http://\xe4\xf6\xfc\xdf.com'), (
             'http://xn--ss-uia6e4a.com', None))
-        self.assertEquals(self.x(
+        self.assertEqual(self.x(
             u'http://visegr\xe1d.com'), ('http://xn--visegrd-mwa.com', None))
-        self.assertEquals(self.x(u'http://h\xe1zipatika.com'), (
+        self.assertEqual(self.x(u'http://h\xe1zipatika.com'), (
             'http://xn--hzipatika-01a.com', None))
-        self.assertEquals(self.x(u'http://www.\xe7ukurova.com'), (
+        self.assertEqual(self.x(u'http://www.\xe7ukurova.com'), (
             'http://www.xn--ukurova-txa.com', None))
-        self.assertEquals(self.x(u'http://nixier\xf6hre.nixieclock-tube.com'), ('http://xn--nixierhre-57a.nixieclock-tube.com', None))
-        self.assertEquals(self.x(u'google.ca.'), ('http://google.ca.', None))
+        self.assertEqual(self.x(u'http://nixier\xf6hre.nixieclock-tube.com'), ('http://xn--nixierhre-57a.nixieclock-tube.com', None))
+        self.assertEqual(self.x(u'google.ca.'), ('http://google.ca.', None))
 
-        self.assertEquals(
+        self.assertEqual(
             self.y(u'https://google.ca'), ('https://google.ca', None))
-        self.assertEquals(self.y(
+        self.assertEqual(self.y(
             u'https://\u4e2d\u4fd4.com'), ('https://xn--fiq13b.com', None))
 
-        self.assertEquals(self.z(u'google.ca'), ('google.ca', None))
+        self.assertEqual(self.z(u'google.ca'), ('google.ca', None))
 
     def testInvalidUrls(self):
-        self.assertEquals(
+        self.assertEqual(
             self.x(u'://ABC.com'), (u'://ABC.com', 'Enter a valid URL'))
-        self.assertEquals(self.x(u'http://\u4e2d\u4fd4.dne'), (
+        self.assertEqual(self.x(u'http://\u4e2d\u4fd4.dne'), (
             u'http://\u4e2d\u4fd4.dne', 'Enter a valid URL'))
-        self.assertEquals(self.x(u'https://google.dne'), (
+        self.assertEqual(self.x(u'https://google.dne'), (
             u'https://google.dne', 'Enter a valid URL'))
-        self.assertEquals(self.x(u'https://google..ca'), (
+        self.assertEqual(self.x(u'https://google..ca'), (
             u'https://google..ca', 'Enter a valid URL'))
-        self.assertEquals(
+        self.assertEqual(
             self.x(u'google..ca'), (u'google..ca', 'Enter a valid URL'))
-        self.assertEquals(self.x(u'http://' + u'\u4e2d' * 1000 + u'.com'), (
+        self.assertEqual(self.x(u'http://' + u'\u4e2d' * 1000 + u'.com'), (
             u'http://' + u'\u4e2d' * 1000 + u'.com', 'Enter a valid URL'))
 
-        self.assertEquals(self.x(u'http://google.com#fragment_\u4e86'), (
+        self.assertEqual(self.x(u'http://google.com#fragment_\u4e86'), (
             u'http://google.com#fragment_\u4e86', 'Enter a valid URL'))
-        self.assertEquals(self.x(u'http\u4e86://google.com'), (
+        self.assertEqual(self.x(u'http\u4e86://google.com'), (
             u'http\u4e86://google.com', 'Enter a valid URL'))
-        self.assertEquals(self.x(u'http\u4e86://google.com#fragment_\u4e86'), (
+        self.assertEqual(self.x(u'http\u4e86://google.com#fragment_\u4e86'), (
             u'http\u4e86://google.com#fragment_\u4e86', 'Enter a valid URL'))
 
-        self.assertEquals(self.y(u'http://\u4e2d\u4fd4.com/\u4e86'), (
+        self.assertEqual(self.y(u'http://\u4e2d\u4fd4.com/\u4e86'), (
             u'http://\u4e2d\u4fd4.com/\u4e86', 'Enter a valid URL'))
-        #self.assertEquals(self.y(u'google.ca'), (u'google.ca', 'Enter a valid URL'))
+        #self.assertEqual(self.y(u'google.ca'), (u'google.ca', 'Enter a valid URL'))
 
-        self.assertEquals(self.z(u'invalid.domain..com'), (
+        self.assertEqual(self.z(u'invalid.domain..com'), (
             u'invalid.domain..com', 'Enter a valid URL'))
-        self.assertEquals(self.z(u'invalid.\u4e2d\u4fd4.blargg'), (
+        self.assertEqual(self.z(u'invalid.\u4e2d\u4fd4.blargg'), (
             u'invalid.\u4e2d\u4fd4.blargg', 'Enter a valid URL'))
 
 # ##############################################################################

gluon/tests/test_validators.py

@@ -235,6 +235,11 @@ class TestValidators(unittest.TestCase):
         self.assertEqual(sorted(rtn), [('%d' % george_id, 'george'), ('%d' % costanza_id, 'costanza')])
         rtn = IS_IN_DB(db, db.person.id, db.person.name, error_message='oops', sort=True).options(zero=True)
         self.assertEqual(rtn, [('', ''), ('%d' % costanza_id, 'costanza'), ('%d' % george_id, 'george')])
+        # Test None
+        rtn = IS_IN_DB(db, 'person.id', '%(name)s', error_message='oops')(None)
+        self.assertEqual(rtn, (None, 'oops'))
+        rtn = IS_IN_DB(db, 'person.name', '%(name)s', error_message='oops')(None)
+        self.assertEqual(rtn, (None, 'oops'))
         # Test using the set it made for options
         vldtr = IS_IN_DB(db, 'person.name', '%(name)s', error_message='oops')
         vldtr.options()
@@ -494,21 +499,23 @@ class TestValidators(unittest.TestCase):
         rtn = IS_NOT_EMPTY()('x')
         self.assertEqual(rtn, ('x', None))
         rtn = IS_NOT_EMPTY()(' x ')
-        self.assertEqual(rtn, ('x', None))
+        self.assertEqual(rtn, (' x ', None))
         rtn = IS_NOT_EMPTY()(None)
         self.assertEqual(rtn, (None, 'Enter a value'))
         rtn = IS_NOT_EMPTY()('')
         self.assertEqual(rtn, ('', 'Enter a value'))
+        rtn = IS_NOT_EMPTY()(b'')
+        self.assertEqual(rtn, (b'', 'Enter a value'))
         rtn = IS_NOT_EMPTY()('  ')
-        self.assertEqual(rtn, ('', 'Enter a value'))
+        self.assertEqual(rtn, ('  ', 'Enter a value'))
         rtn = IS_NOT_EMPTY()(' \n\t')
-        self.assertEqual(rtn, ('', 'Enter a value'))
+        self.assertEqual(rtn, (' \n\t', 'Enter a value'))
         rtn = IS_NOT_EMPTY()([])
         self.assertEqual(rtn, ([], 'Enter a value'))
         rtn = IS_NOT_EMPTY(empty_regex='def')('def')
-        self.assertEqual(rtn, ('', 'Enter a value'))
+        self.assertEqual(rtn, ('def', 'Enter a value'))
         rtn = IS_NOT_EMPTY(empty_regex='de[fg]')('deg')
-        self.assertEqual(rtn, ('', 'Enter a value'))
+        self.assertEqual(rtn, ('deg', 'Enter a value'))
         rtn = IS_NOT_EMPTY(empty_regex='def')('abc')
         self.assertEqual(rtn, ('abc', None))
 
@@ -591,6 +598,11 @@ class TestValidators(unittest.TestCase):
         rtn = IS_EMAIL(error_message='oops')(42)
         self.assertEqual(rtn, (42, 'oops'))
 
+        # test for Internationalized Domain Names, see https://docs.python.org/2/library/codecs.html#module-encodings.idna
+        rtn = IS_EMAIL()('web2py@Alliancefrançaise.nu')
+        self.assertEqual(rtn, ('web2py@Alliancefrançaise.nu', None))
+
+
     def test_IS_LIST_OF_EMAILS(self):
         emails = ['localguy@localhost', '_Yosemite.Sam@example.com']
         rtn = IS_LIST_OF_EMAILS()(','.join(emails))
@@ -762,15 +774,19 @@ class TestValidators(unittest.TestCase):
 
     def test_IS_LOWER(self):
         rtn = IS_LOWER()('ABC')
+        self.assertEqual(rtn, ('abc', None))
+        rtn = IS_LOWER()(b'ABC')
         self.assertEqual(rtn, (b'abc', None))
         rtn = IS_LOWER()('Ñ')
-        self.assertEqual(rtn, (b'\xc3\xb1', None))
+        self.assertEqual(rtn, ('ñ', None))
 
     def test_IS_UPPER(self):
         rtn = IS_UPPER()('abc')
+        self.assertEqual(rtn, ('ABC', None))
+        rtn = IS_UPPER()(b'abc')
         self.assertEqual(rtn, (b'ABC', None))
         rtn = IS_UPPER()('ñ')
-        self.assertEqual(rtn, (b'\xc3\x91', None))
+        self.assertEqual(rtn, ('Ñ', None))
 
     def test_IS_SLUG(self):
         rtn = IS_SLUG()('abc123')
@@ -840,7 +856,7 @@ class TestValidators(unittest.TestCase):
         rtn = IS_EMPTY_OR(IS_EMAIL())('abc')
         self.assertEqual(rtn, ('abc', 'Enter a valid email address'))
         rtn = IS_EMPTY_OR(IS_EMAIL())(' abc ')
-        self.assertEqual(rtn, ('abc', 'Enter a valid email address'))
+        self.assertEqual(rtn, (' abc ', 'Enter a valid email address'))
         rtn = IS_EMPTY_OR(IS_IN_SET([('id1', 'first label'), ('id2', 'second label')], zero='zero')).options(zero=False)
         self.assertEqual(rtn, [('', ''), ('id1', 'first label'), ('id2', 'second label')])
         rtn = IS_EMPTY_OR(IS_IN_SET([('id1', 'first label'), ('id2', 'second label')], zero='zero')).options()

gluon/tests/test_web.py

@@ -110,7 +110,7 @@ class TestWeb(LiveTest):
         self.assertTrue('Welcome Homer' in client.text)
 
         client = WebClient('http://127.0.0.1:8000/admin/default/')
-        client.post('index', data=dict(password='hello'))
+        client.post('index', data=dict(password='testpass'))
         client.get('site')
         client.get('design/welcome')
 

gluon/utils.py

@@ -156,12 +156,12 @@ def get_digest(value):
         raise ValueError("Invalid digest algorithm: %s" % value)
 
 DIGEST_ALG_BY_SIZE = {
-    128 / 4: 'md5',
-    160 / 4: 'sha1',
-    224 / 4: 'sha224',
-    256 / 4: 'sha256',
-    384 / 4: 'sha384',
-    512 / 4: 'sha512',
+    128 // 4: 'md5',
+    160 // 4: 'sha1',
+    224 // 4: 'sha224',
+    256 // 4: 'sha256',
+    384 // 4: 'sha384',
+    512 // 4: 'sha512',
 }
 
 

gluon/validators.py

@@ -10,7 +10,6 @@
 Validators
 -----------
 """
-
 import os
 import re
 import datetime
@@ -21,7 +20,9 @@ import urllib
 import struct
 import decimal
 import unicodedata
-from gluon._compat import StringIO, long, unicodeT, to_unicode, urllib_unquote, unichr, to_bytes, PY2, to_unicode, to_native
+
+from gluon._compat import StringIO, long, basestring, unicodeT, to_unicode, urllib_unquote, unichr, to_bytes, PY2, \
+    to_unicode, to_native, string_types, urlparse
 from gluon.utils import simple_hash, web2py_uuid, DIGEST_ALG_BY_SIZE
 from pydal.objects import Field, FieldVirtual, FieldMethod
 from functools import reduce
@@ -195,7 +196,7 @@ class IS_MATCH(Validator):
         self.is_unicode = is_unicode or (not(PY2))
 
     def __call__(self, value):
-        if not(PY2): # PY3 convert bytes to unicode
+        if not(PY2):  # PY3 convert bytes to unicode
             value = to_unicode(value)
 
         if self.is_unicode or not(PY2):
@@ -270,7 +271,7 @@ class IS_EXPR(Validator):
             return (value, self.expression(value))
         # for backward compatibility
         self.environment.update(value=value)
-        exec ('__ret__=' + self.expression, self.environment)
+        exec('__ret__=' + self.expression, self.environment)
         if self.environment['__ret__']:
             return (value, None)
         return (value, translate(self.error_message))
@@ -452,10 +453,10 @@ class IS_IN_SET(Validator):
         if not self.labels:
             items = [(k, k) for (i, k) in enumerate(self.theset)]
         else:
-            items = [(k, self.labels[i]) for (i, k) in enumerate(self.theset)]
+            items = [(k, list(self.labels)[i]) for (i, k) in enumerate(self.theset)]
         if self.sort:
             items.sort(key=lambda o: str(o[1]).upper())
-        if zero and not self.zero is None and not self.multiple:
+        if zero and self.zero is not None and not self.multiple:
             items.insert(0, ('', self.zero))
         return items
 
@@ -659,7 +660,7 @@ class IS_IN_DB(Validator):
                     return (values, None)
         else:
             if field.type in ('id', 'integer'):
-                if isinstance(value, (int, long)) or value.isdigit():
+                if isinstance(value, (int, long)) or (isinstance(value, string_types) and value.isdigit()):
                     value = int(value)
                 elif self.auto_add:
                     value = self.maybe_add(table, self.fieldnames[0], value)
@@ -823,7 +824,7 @@ class IS_INT_IN_RANGE(Validator):
 
 def str2dec(number):
     s = str(number)
-    if not '.' in s:
+    if '.' not in s:
         s += '.00'
     else:
         s += '0' * (2 - len(s.split('.')[1]))
@@ -989,14 +990,15 @@ class IS_DECIMAL_IN_RANGE(Validator):
 
 
 def is_empty(value, empty_regex=None):
+    _value = value
     """test empty field"""
     if isinstance(value, (str, unicodeT)):
         value = value.strip()
         if empty_regex is not None and empty_regex.match(value):
             value = ''
-    if value is None or value == '' or value == []:
-        return (value, True)
-    return (value, False)
+    if value is None or value == '' or value == b'' or value == []:
+        return (_value, True)
+    return (_value, False)
 
 
 class IS_NOT_EMPTY(Validator):
@@ -1156,15 +1158,16 @@ class IS_EMAIL(Validator):
 
     """
 
-    regex = re.compile('''
+    body_regex = re.compile('''
         ^(?!\.)                            # name may not begin with a dot
         (
           [-a-z0-9!\#$%&'*+/=?^_`{|}~]     # all legal characters except dot
           |
           (?<!\.)\.                        # single dots only
         )+
-        (?<!\.)                            # name may not end with a dot
-        @
+        (?<!\.)$                            # name may not end with a dot
+    ''', re.VERBOSE | re.IGNORECASE)
+    domain_regex = re.compile('''
         (
           localhost
           |
@@ -1196,14 +1199,27 @@ class IS_EMAIL(Validator):
         self.error_message = error_message
 
     def __call__(self, value):
+        if not(isinstance(value, (basestring, unicodeT))) or not value or '@' not in value:
+            return (value, translate(self.error_message))
+
+        body, domain = value.rsplit('@', 1)
+
         try:
-            match = self.regex.match(value)
-        except TypeError:
+            match_body = self.body_regex.match(body)
+            match_domain = self.domain_regex.match(domain)
+
+            if not match_domain:
+                # check for Internationalized Domain Names
+                # see https://docs.python.org/2/library/codecs.html#module-encodings.idna
+                domain_encoded = to_unicode(domain).encode('idna').decode('ascii')
+                match_domain = self.domain_regex.match(domain_encoded)
+
+            match = (match_body is not None) and (match_domain is not None)
+        except (TypeError, UnicodeError):
             # Value may not be a string where we can look for matches.
             # Example: we're calling ANY_OF formatter and IS_EMAIL is asked to validate a date.
             match = None
         if match:
-            domain = value.split('@')[1]
             if (not self.banned or not self.banned.match(domain)) \
                     and (not self.forced or self.forced.match(domain)):
                 return (value, None)
@@ -1232,7 +1248,7 @@ class IS_LIST_OF_EMAILS(object):
         f = IS_EMAIL()
         for email in self.split_emails.findall(value):
             error = f(email)[1]
-            if error and not email in bad_emails:
+            if error and email not in bad_emails:
                 bad_emails.append(email)
         if not bad_emails:
             return (value, None)
@@ -1372,19 +1388,6 @@ unofficial_url_schemes = [
 all_url_schemes = [None] + official_url_schemes + unofficial_url_schemes
 http_schemes = [None, 'http', 'https']
 
-
-# This regex comes from RFC 2396, Appendix B. It's used to split a URL into
-# its component parts
-# Here are the regex groups that it extracts:
-#    scheme = group(2)
-#    authority = group(4)
-#    path = group(5)
-#    query = group(7)
-#    fragment = group(9)
-
-url_split_regex = \
-    re.compile('^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?')
-
 # Defined in RFC 3490, Section 3.1, Requirement #1
 # Use this regex to split the authority component of a unicode URL into
 # its component labels
@@ -1454,18 +1457,15 @@ def unicode_to_ascii_authority(authority):
     # We use the ToASCII operation because we are about to put the authority
     # into an IDN-unaware slot
     asciiLabels = []
-    try:
-        import encodings.idna
-        for label in labels:
-            if label:
-                asciiLabels.append(to_native(encodings.idna.ToASCII(label)))
-            else:
-                 # encodings.idna.ToASCII does not accept an empty string, but
-                 # it is necessary for us to allow for empty labels so that we
-                 # don't modify the URL
-                asciiLabels.append('')
-    except:
-        asciiLabels = [str(label) for label in labels]
+    import encodings.idna
+    for label in labels:
+        if label:
+            asciiLabels.append(to_native(encodings.idna.ToASCII(label)))
+        else:
+            # encodings.idna.ToASCII does not accept an empty string, but
+            # it is necessary for us to allow for empty labels so that we
+            # don't modify the URL
+            asciiLabels.append('')
     # RFC 3490, Section 4, Step 5
     return str(reduce(lambda x, y: x + unichr(0x002E) + y, asciiLabels))
 
@@ -1504,33 +1504,39 @@ def unicode_to_ascii_url(url, prepend_scheme):
     """
     # convert the authority component of the URL into an ASCII punycode string,
     # but encode the rest using the regular URI character encoding
-
-    groups = url_split_regex.match(url).groups()
+    components = urlparse.urlparse(url)
+    prepended = False
     # If no authority was found
-    if not groups[3]:
+    if not components.netloc:
         # Try appending a scheme to see if that fixes the problem
         scheme_to_prepend = prepend_scheme or 'http'
-        groups = url_split_regex.match(
-            to_unicode(scheme_to_prepend) + u'://' + url).groups()
+        components = urlparse.urlparse(to_unicode(scheme_to_prepend) + u'://' + url)
+        prepended = True
+
     # if we still can't find the authority
-    if not groups[3]:
+    if not components.netloc:
         raise Exception('No authority component found, ' +
                         'could not decode unicode to US-ASCII')
 
     # We're here if we found an authority, let's rebuild the URL
-    scheme = groups[1]
-    authority = groups[3]
-    path = groups[4] or ''
-    query = groups[5] or ''
-    fragment = groups[7] or ''
+    scheme = components.scheme
+    authority = components.netloc
+    path = components.path
+    query = components.query
+    fragment = components.fragment
 
-    if prepend_scheme:
-        scheme = str(scheme) + '://'
-    else:
+    if prepended:
         scheme = ''
 
-    return scheme + unicode_to_ascii_authority(authority) +\
-        escape_unicode(path) + escape_unicode(query) + str(fragment)
+    unparsed = urlparse.urlunparse((scheme,
+                                    unicode_to_ascii_authority(authority),
+                                    escape_unicode(path),
+                                    '',
+                                    escape_unicode(query),
+                                    str(fragment)))
+    if unparsed.startswith('//'):
+        unparsed = unparsed[2:]  # Remove the // urlunparse puts in the beginning
+    return unparsed
 
 
 class IS_GENERIC_URL(Validator):
@@ -1591,7 +1597,8 @@ class IS_GENERIC_URL(Validator):
                               % (self.prepend_scheme, self.allowed_schemes))
 
     GENERIC_URL = re.compile(r"%[^0-9A-Fa-f]{2}|%[^0-9A-Fa-f][0-9A-Fa-f]|%[0-9A-Fa-f][^0-9A-Fa-f]|%$|%[0-9A-Fa-f]$|%[^0-9A-Fa-f]$")
-    GENERIC_URL_VALID = re.compile(r"[A-Za-z0-9;/?:@&=+$,\-_\.!~*'\(\)%#]+$")
+    GENERIC_URL_VALID = re.compile(r"[A-Za-z0-9;/?:@&=+$,\-_\.!~*'\(\)%]+$")
+    URL_FRAGMENT_VALID = re.compile(r"[|A-Za-z0-9;/?:@&=+$,\-_\.!~*'\(\)%]+$")
 
     def __call__(self, value):
         """
@@ -1603,41 +1610,49 @@ class IS_GENERIC_URL(Validator):
             prepended with prepend_scheme), and tuple[1] is either
             None (success!) or the string error_message
         """
-        try:
-            # if the URL does not misuse the '%' character
-            if not self.GENERIC_URL.search(value):
-                # if the URL is only composed of valid characters
-                if self.GENERIC_URL_VALID.match(value):
-                    # Then split up the URL into its components and check on
-                    # the scheme
-                    scheme = url_split_regex.match(value).group(2)
-                    # Clean up the scheme before we check it
-                    if not scheme is None:
-                        scheme = urllib_unquote(scheme).lower()
-                    # If the scheme really exists
-                    if scheme in self.allowed_schemes:
-                        # Then the URL is valid
-                        return (value, None)
-                    else:
-                        # else, for the possible case of abbreviated URLs with
-                        # ports, check to see if adding a valid scheme fixes
-                        # the problem (but only do this if it doesn't have
-                        # one already!)
-                        if value.find('://') < 0 and None in self.allowed_schemes:
-                            schemeToUse = self.prepend_scheme or 'http'
-                            prependTest = self.__call__(
-                                schemeToUse + '://' + value)
-                            # if the prepend test succeeded
-                            if prependTest[1] is None:
-                                # if prepending in the output is enabled
-                                if self.prepend_scheme:
-                                    return prependTest
-                                else:
-                                    # else return the original,
-                                    #  non-prepended value
-                                    return (value, None)
-        except:
-            pass
+
+        # if we dont have anything or the URL misuses the '%' character
+
+        if not value or self.GENERIC_URL.search(value):
+            return (value, translate(self.error_message))
+
+        if '#' in value:
+            url, fragment_part = value.split('#')
+        else:
+            url, fragment_part = value, ''
+        # if the URL is only composed of valid characters
+        if self.GENERIC_URL_VALID.match(url) and (not fragment_part or self.URL_FRAGMENT_VALID.match(fragment_part)):
+            # Then parse the URL into its components and check on
+            try:
+                components = urlparse.urlparse(urllib_unquote(value))._asdict()
+            except ValueError:
+                return (value, translate(self.error_message))
+
+            # Clean up the scheme before we check it
+            scheme = components['scheme']
+            if len(scheme) == 0:
+                scheme = None
+            else:
+                scheme = components['scheme'].lower()
+            # If the scheme doesn't really exists
+            if scheme not in self.allowed_schemes or not scheme and ':' in components['path']:
+                # for the possible case of abbreviated URLs with
+                # ports, check to see if adding a valid scheme fixes
+                # the problem (but only do this if it doesn't have
+                # one already!)
+                if '://' not in value and None in self.allowed_schemes:
+                    schemeToUse = self.prepend_scheme or 'http'
+                    prependTest = self.__call__(
+                        schemeToUse + '://' + value)
+                    # if the prepend test succeeded
+                    if prependTest[1] is None:
+                        # if prepending in the output is enabled
+                        if self.prepend_scheme:
+                            return prependTest
+                        else:
+                            return (value, None)
+            else:
+                return (value, None)
         # else the URL is not valid
         return (value, translate(self.error_message))
 
@@ -1904,15 +1919,14 @@ class IS_HTTP_URL(Validator):
             (possible prepended with prepend_scheme), and tuple[1] is either
             None (success!) or the string error_message
         """
-
         try:
             # if the URL passes generic validation
             x = IS_GENERIC_URL(error_message=self.error_message,
                                allowed_schemes=self.allowed_schemes,
                                prepend_scheme=self.prepend_scheme)
             if x(value)[1] is None:
-                componentsMatch = url_split_regex.match(value)
-                authority = componentsMatch.group(4)
+                components = urlparse.urlparse(value)
+                authority = components.netloc
                 # if there is an authority component
                 if authority:
                     # if authority is a valid IP address
@@ -1932,7 +1946,7 @@ class IS_HTTP_URL(Validator):
                 else:
                     # else this is a relative/abbreviated URL, which will parse
                     # into the URL's path component
-                    path = componentsMatch.group(5)
+                    path = components.path
                     # relative case: if this is a valid path (if it starts with
                     # a slash)
                     if path.startswith('/'):
@@ -1941,7 +1955,7 @@ class IS_HTTP_URL(Validator):
                     else:
                         # abbreviated case: if we haven't already, prepend a
                         # scheme and see if it fixes the problem
-                        if value.find('://') < 0:
+                        if '://' not in value and None in self.allowed_schemes:
                             schemeToUse = self.prepend_scheme or 'http'
                             prependTest = self.__call__(schemeToUse
                                                         + '://' + value)
@@ -2108,7 +2122,6 @@ class IS_URL(Validator):
                 # If we are not able to convert the unicode url into a
                 # US-ASCII URL, then the URL is not valid
                 return (value, translate(self.error_message))
-
             methodResult = subMethod(asciiValue)
             # if the validation of the US-ASCII version of the value failed
             if not methodResult[1] is None:
@@ -2480,7 +2493,13 @@ class IS_LOWER(Validator):
     """
 
     def __call__(self, value):
-        return (to_bytes(to_unicode(value).lower()), None)
+        cast_back = lambda x: x
+        if isinstance(value, str):
+            cast_back = to_native
+        elif isinstance(value, bytes):
+            cast_back = to_bytes
+        value = to_unicode(value).lower()
+        return (cast_back(value), None)
 
 
 class IS_UPPER(Validator):
@@ -2495,7 +2514,13 @@ class IS_UPPER(Validator):
     """
 
     def __call__(self, value):
-        return (to_bytes(to_unicode(value).upper()), None)
+        cast_back = lambda x: x
+        if isinstance(value, str):
+            cast_back = to_native
+        elif isinstance(value, bytes):
+            cast_back = to_bytes
+        value = to_unicode(value).upper()
+        return (cast_back(value), None)
 
 
 def urlify(s, maxlen=80, keep_underscores=False):
@@ -2602,7 +2627,7 @@ class ANY_OF(Validator):
     def __call__(self, value):
         for validator in self.subs:
             value, error = validator(value)
-            if error == None:
+            if error is None:
                 break
         return value, error
 
@@ -2742,7 +2767,7 @@ class LazyCrypt(object):
         else:
             digest_alg, key = self.crypt.digest_alg, ''
         if self.crypt.salt:
-            if self.crypt.salt == True:
+            if self.crypt.salt is True:
                 salt = str(web2py_uuid()).replace('-', '')[-16:]
             else:
                 salt = self.crypt.salt
@@ -2827,7 +2852,7 @@ class CRYPT(object):
     Supports standard algorithms
 
         >>> for alg in ('md5','sha1','sha256','sha384','sha512'):
-        ...     print str(CRYPT(digest_alg=alg,salt=True)('test')[0])
+        ...     print(str(CRYPT(digest_alg=alg,salt=True)('test')[0]))
         md5$...$...
         sha1$...$...
         sha256$...$...
@@ -2839,13 +2864,13 @@ class CRYPT(object):
     Supports for pbkdf2
 
         >>> alg = 'pbkdf2(1000,20,sha512)'
-        >>> print str(CRYPT(digest_alg=alg,salt=True)('test')[0])
+        >>> print(str(CRYPT(digest_alg=alg,salt=True)('test')[0]))
         pbkdf2(1000,20,sha512)$...$...
 
     An optional hmac_key can be specified and it is used as salt prefix
 
         >>> a = str(CRYPT(digest_alg='md5',key='mykey',salt=True)('test')[0])
-        >>> print a
+        >>> print(a)
         md5$...$...
 
     Even if the algorithm changes the hash can still be validated

gluon/widget.py

@@ -13,7 +13,7 @@ from __future__ import print_function
 
 import datetime
 import sys
-from gluon._compat import StringIO, thread, xrange
+from gluon._compat import StringIO, thread, xrange, PY2
 import time
 import threading
 import os
@@ -134,24 +134,29 @@ class web2pyDialog(object):
     def __init__(self, root, options):
         """ web2pyDialog constructor  """
 
-        import Tkinter
-        import tkMessageBox
+        if PY2:
+            import Tkinter as tkinter
+            import tkMessageBox as messagebox
+        else:
+            import tkinter
+            from tkinter import messagebox
+
 
         bg_color = 'white'
         root.withdraw()
 
-        self.root = Tkinter.Toplevel(root, bg=bg_color)
+        self.root = tkinter.Toplevel(root, bg=bg_color)
         self.root.resizable(0, 0)
         self.root.title(ProgramName)
 
         self.options = options
         self.scheduler_processes = {}
-        self.menu = Tkinter.Menu(self.root)
-        servermenu = Tkinter.Menu(self.menu, tearoff=0)
+        self.menu = tkinter.Menu(self.root)
+        servermenu = tkinter.Menu(self.menu, tearoff=0)
         httplog = os.path.join(self.options.folder, self.options.log_filename)
         iconphoto = os.path.join('extras', 'icons', 'web2py.gif')
         if os.path.exists(iconphoto):
-            img = Tkinter.PhotoImage(file=iconphoto)
+            img = tkinter.PhotoImage(file=iconphoto)
             self.root.tk.call('wm', 'iconphoto', self.root._w, img)
         # Building the Menu
         item = lambda: start_browser(httplog)
@@ -163,16 +168,16 @@ class web2pyDialog(object):
 
         self.menu.add_cascade(label='Server', menu=servermenu)
 
-        self.pagesmenu = Tkinter.Menu(self.menu, tearoff=0)
+        self.pagesmenu = tkinter.Menu(self.menu, tearoff=0)
         self.menu.add_cascade(label='Pages', menu=self.pagesmenu)
 
         #scheduler menu
-        self.schedmenu = Tkinter.Menu(self.menu, tearoff=0)
+        self.schedmenu = tkinter.Menu(self.menu, tearoff=0)
         self.menu.add_cascade(label='Scheduler', menu=self.schedmenu)
         #start and register schedulers from options
         self.update_schedulers(start=True)
 
-        helpmenu = Tkinter.Menu(self.menu, tearoff=0)
+        helpmenu = tkinter.Menu(self.menu, tearoff=0)
 
         # Home Page
         item = lambda: start_browser('http://www.web2py.com/')
@@ -180,7 +185,7 @@ class web2pyDialog(object):
                              command=item)
 
         # About
-        item = lambda: tkMessageBox.showinfo('About web2py', ProgramInfo)
+        item = lambda: messagebox.showinfo('About web2py', ProgramInfo)
         helpmenu.add_command(label='About',
                              command=item)
 
@@ -194,10 +199,10 @@ class web2pyDialog(object):
         else:
             self.root.protocol('WM_DELETE_WINDOW', self.quit)
 
-        sticky = Tkinter.NW
+        sticky = tkinter.NW
 
         # Prepare the logo area
-        self.logoarea = Tkinter.Canvas(self.root,
+        self.logoarea = tkinter.Canvas(self.root,
                                 background=bg_color,
                                 width=300,
                                 height=300)
@@ -206,22 +211,22 @@ class web2pyDialog(object):
 
         logo = os.path.join('extras', 'icons', 'splashlogo.gif')
         if os.path.exists(logo):
-            img = Tkinter.PhotoImage(file=logo)
-            pnl = Tkinter.Label(self.logoarea, image=img, background=bg_color, bd=0)
+            img = tkinter.PhotoImage(file=logo)
+            pnl = tkinter.Label(self.logoarea, image=img, background=bg_color, bd=0)
             pnl.pack(side='top', fill='both', expand='yes')
             # Prevent garbage collection of img
             pnl.image = img
 
         # Prepare the banner area
-        self.bannerarea = Tkinter.Canvas(self.root,
+        self.bannerarea = tkinter.Canvas(self.root,
                                 bg=bg_color,
                                 width=300,
                                 height=300)
         self.bannerarea.grid(row=1, column=1, columnspan=2, sticky=sticky)
 
-        Tkinter.Label(self.bannerarea, anchor=Tkinter.N,
+        tkinter.Label(self.bannerarea, anchor=tkinter.N,
                       text=str(ProgramVersion + "\n" + ProgramAuthor),
-                      font=('Helvetica', 11), justify=Tkinter.CENTER,
+                      font=('Helvetica', 11), justify=tkinter.CENTER,
                       foreground='#195866', background=bg_color,
                       height=3).pack(side='top',
                                      fill='both',
@@ -230,24 +235,24 @@ class web2pyDialog(object):
         self.bannerarea.after(1000, self.update_canvas)
 
         # IP
-        Tkinter.Label(self.root,
+        tkinter.Label(self.root,
                       text='Server IP:', bg=bg_color,
-                      justify=Tkinter.RIGHT).grid(row=4,
+                      justify=tkinter.RIGHT).grid(row=4,
                                                   column=1,
                                                   sticky=sticky)
         self.ips = {}
-        self.selected_ip = Tkinter.StringVar()
+        self.selected_ip = tkinter.StringVar()
         row = 4
         ips = [('127.0.0.1', 'Local (IPv4)')] + \
             ([('::1', 'Local (IPv6)')] if socket.has_ipv6 else []) + \
             [(ip, 'Public') for ip in options.ips] + \
             [('0.0.0.0', 'Public')]
         for ip, legend in ips:
-            self.ips[ip] = Tkinter.Radiobutton(
+            self.ips[ip] = tkinter.Radiobutton(
                 self.root, bg=bg_color, highlightthickness=0,
                 selectcolor='light grey', width=30,
-                anchor=Tkinter.W, text='%s (%s)' % (legend, ip),
-                justify=Tkinter.LEFT,
+                anchor=tkinter.W, text='%s (%s)' % (legend, ip),
+                justify=tkinter.LEFT,
                 variable=self.selected_ip, value=ip)
             self.ips[ip].grid(row=row, column=2, sticky=sticky)
             if row == 4:
@@ -256,30 +261,30 @@ class web2pyDialog(object):
         shift = row
 
         # Port
-        Tkinter.Label(self.root,
+        tkinter.Label(self.root,
                       text='Server Port:', bg=bg_color,
-                      justify=Tkinter.RIGHT).grid(row=shift,
+                      justify=tkinter.RIGHT).grid(row=shift,
                                                   column=1, pady=10,
                                                   sticky=sticky)
 
-        self.port_number = Tkinter.Entry(self.root)
-        self.port_number.insert(Tkinter.END, self.options.port)
+        self.port_number = tkinter.Entry(self.root)
+        self.port_number.insert(tkinter.END, self.options.port)
         self.port_number.grid(row=shift, column=2, sticky=sticky, pady=10)
 
         # Password
-        Tkinter.Label(self.root,
+        tkinter.Label(self.root,
                       text='Choose Password:', bg=bg_color,
-                      justify=Tkinter.RIGHT).grid(row=shift + 1,
+                      justify=tkinter.RIGHT).grid(row=shift + 1,
                                                   column=1,
                                                   sticky=sticky)
 
-        self.password = Tkinter.Entry(self.root, show='*')
+        self.password = tkinter.Entry(self.root, show='*')
         self.password.bind('<Return>', lambda e: self.start())
         self.password.focus_force()
         self.password.grid(row=shift + 1, column=2, sticky=sticky)
 
         # Prepare the canvas
-        self.canvas = Tkinter.Canvas(self.root,
+        self.canvas = tkinter.Canvas(self.root,
                                      width=400,
                                      height=100,
                                      bg='black')
@@ -288,19 +293,19 @@ class web2pyDialog(object):
         self.canvas.after(1000, self.update_canvas)
 
         # Prepare the frame
-        frame = Tkinter.Frame(self.root)
+        frame = tkinter.Frame(self.root)
         frame.grid(row=shift + 3, column=1, columnspan=2, pady=5,
                    sticky=sticky)
 
         # Start button
-        self.button_start = Tkinter.Button(frame,
+        self.button_start = tkinter.Button(frame,
                                            text='start server',
                                            command=self.start)
 
         self.button_start.grid(row=0, column=0, sticky=sticky)
 
         # Stop button
-        self.button_stop = Tkinter.Button(frame,
+        self.button_stop = tkinter.Button(frame,
                                           text='stop server',
                                           command=self.stop)
 
@@ -454,9 +459,12 @@ class web2pyDialog(object):
 
     def error(self, message):
         """ Shows error message """
+        if PY2:
+            import tkMessageBox as messagebox
+        else:
+            from tkinter import messagebox
 
-        import tkMessageBox
-        tkMessageBox.showerror('web2py start server', message)
+        messagebox.showerror('web2py start server', message)
 
     def start(self):
         """ Starts web2py server """
@@ -1068,7 +1076,10 @@ def start_schedulers(options):
         return
 
     # Work around OS X problem: http://bugs.python.org/issue9405
-    import urllib
+    if PY2:
+        import urllib
+    else:
+        import urllib.request as urllib
     urllib.getproxies()
 
     for app in apps:
@@ -1191,10 +1202,13 @@ def start(cron=True):
 
     if not options.nogui and options.password == '<ask>':
         try:
-            import Tkinter
+            if PY2:
+                import Tkinter as tkinter
+            else:
+                import tkinter
             havetk = True
             try:
-                root = Tkinter.Tk()
+                root = tkinter.Tk()
             except:
                 pass
         except (ImportError, OSError):

scripts/cpdb.py

@@ -15,6 +15,7 @@ except ImportError:
   import readline
 try:
     from gluon import DAL
+    from gluon.fileutils import open_file
 except ImportError as err:
     print('gluon path not found')
 
@@ -531,7 +532,7 @@ class setCopyDB():
         self.db.export_to_csv_file(open('tmp.sql', 'wb'))
 
         print 'importing data...'
-        other_db.import_from_csv_file(open('tmp.sql', 'rb'))
+        other_db.import_from_csv_file(open_file('tmp.sql', 'rb'))
         other_db.commit()
         print 'done!'
         print 'Attention: do not run this program again or you end up with duplicate records'

scripts/setup-web2py-nginx-uwsgi-ubuntu.sh

@@ -222,7 +222,7 @@ echo <<EOF
 you can stop uwsgi and nginx with
 
   sudo /etc/init.d/nginx stop
-  sudo systemctl start emperor.uwsgi.service
+  sudo systemctl stop emperor.uwsgi.service
  
 and start it with