restoring commented tests

[Py3] Open shell executed files in binary mode, fixes #2060

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,35 @@
+---
+name: Bug report
+about: Create a report to help us improve
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Go to '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Desktop (please complete the following information):**
+ - OS: [e.g. iOS]
+ - Browser [e.g. chrome, safari]
+ - Version [e.g. 22]
+
+**Smartphone (please complete the following information):**
+ - Device: [e.g. iPhone6]
+ - OS: [e.g. iOS8.1]
+ - Browser [e.g. stock browser, safari]
+ - Version [e.g. 22]
+
+**Additional context**
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,17 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

CHANGELOG

@@ -1,4 +1,4 @@
-## 2.17.1
+## 2.17.1-2
 - pydal 18.08
 - many small bug fixes
 

Makefile

@@ -44,7 +44,7 @@ rmfiles:
 	rm -rf applications/examples/uploads/* 
 src:
 	### Use semantic versioning
-	echo 'Version 2.17.1-stable+timestamp.'`date +%Y.%m.%d.%H.%M.%S` > VERSION
+	echo 'Version 2.17.2-stable+timestamp.'`date +%Y.%m.%d.%H.%M.%S` > VERSION
 	### rm -f all junk files
 	#make clean
 	# make rmfiles

VERSION

@@ -1 +1 @@
-Version 2.17.1-stable+timestamp.2018.08.05.17.57.00
+Version 2.17.2-stable+timestamp.2018.10.06.11.34.06

anyserver.py

@@ -212,16 +212,16 @@ def mongrel2_handler(application, conn, debug=False):
 
     while True:
         if debug:
-            print "WAITING FOR REQUEST"
+            print("WAITING FOR REQUEST")
 
         # receive a request
         req = conn.recv()
         if debug:
-            print "REQUEST BODY: %r\n" % req.body
+            print("REQUEST BODY: %r\n" % req.body)
 
         if req.is_disconnect():
             if debug:
-                print "DISCONNECT"
+                print("DISCONNECT")
             continue  # effectively ignore the disconnect from the client
 
         # Set a couple of environment attributes a.k.a. header attributes
@@ -247,7 +247,7 @@ def mongrel2_handler(application, conn, debug=False):
         environ['wsgi.input'] = req.body
 
         if debug:
-            print "ENVIRON: %r\n" % environ
+            print("ENVIRON: %r\n" % environ)
 
         # SimpleHandler needs file-like stream objects for
         # requests, errors and responses
@@ -282,10 +282,10 @@ def mongrel2_handler(application, conn, debug=False):
 
         # return the response
         if debug:
-            print "RESPONSE: %r\n" % response
+            print("RESPONSE: %r\n" % response)
         if errors:
             if debug:
-                print "ERRORS: %r" % errors
+                print("ERRORS: %r" % errors)
             data = "%s\r\n\r\n%s" % (data, errors)
         conn.reply_http(
             req, data, code=code, status=status, headers=headers)
@@ -355,8 +355,8 @@ def main():
                       dest='workers',
                       help='number of workers number')
     (options, args) = parser.parse_args()
-    print 'starting %s on %s:%s...' % (
+    print('starting %s on %s:%s...' % (
-        options.server, options.ip, options.port)
+        options.server, options.ip, options.port))
     run(options.server, options.ip, options.port,
         logging=options.logging, profiler=options.profiler_dir,
         options=options)

applications/admin/controllers/default.py

@@ -562,7 +562,11 @@ def enable():
         os.unlink(filename)
         return SPAN(T('Disable'), _style='color:green')
     else:
-        safe_open(filename, 'wb').write('disabled: True\ntime-disabled: %s' % request.now)
+        if PY2:
+            safe_open(filename, 'wb').write('disabled: True\ntime-disabled: %s' % request.now)
+        else:
+            str_ = 'disabled: True\ntime-disabled: %s' % request.now
+            safe_open(filename, 'wb').write(str_.encode('utf-8'))
         return SPAN(T('Enable'), _style='color:red')
 
 
@@ -642,7 +646,10 @@ def edit():
     # show settings tab and save prefernces
     if 'settings' in request.vars:
         if request.post_vars:  # save new preferences
-            post_vars = request.post_vars.items()
+            if PY2:
+                post_vars = request.post_vars.items()
+            else:
+                post_vars = list(request.post_vars.items())
             # Since unchecked checkbox are not serialized, we must set them as false by hand to store the correct preference in the settings
             post_vars += [(opt, 'false') for opt in preferences if opt not in request.post_vars]
             if config.save(post_vars):

applications/admin/static/js/web2py_bootstrap.js

@@ -29,5 +29,14 @@ jQuery(function(){
   }
   hoverMenu(); // first page load
   jQuery(window).resize(hoverMenu); // on resize event
-  jQuery('ul.nav li.dropdown a').click(function(){window.location=jQuery(this).attr('href');});
+  jQuery('ul.nav li.dropdown a').click(function(){ 
+    if(jQuery(this).attr("target")){
+        window.open(
+            jQuery(this).attr('href'),
+            jQuery(this).attr('target') // <- This is what makes it open in a new window.
+        );
+    } else {
+        window.location=jQuery(this).attr('href'); 
+    }
+  });
 });

applications/admin/views/default/plugin.html

@@ -85,7 +85,7 @@ def deletefile(arglist):
 				{{=peekfile('models',m)}}
 			</span>
 			<span class="extras">
-				{{if len(defines[m]):}}{{=T("defines tables")}} {{pass}}{{=XML(', '.join([B(table).xml() for table in defines[m]]))}}
+				{{if len(defines[m]):}}{{=T("defines tables")}} {{pass}}{{=XML(b', '.join([B(table).xml() for table in defines[m]]))}}
 			</span>
 		</li>
 		{{pass}}
@@ -118,7 +118,7 @@ for c in controllers: controller_functions+=[c[:-3]+'/%s.html'%x for x in functi
 				{{=peekfile('controllers',c)}}
 			</span>
 			<span class="extras celled">
-				{{if functions[c]:}}{{=T("exposes")}} {{pass}}{{=XML(', '.join([A(f,_href=URL(a=app,c=c[:-3],f=f)).xml() for f in functions[c]]))}}
+				{{if functions[c]:}}{{=T("exposes")}} {{pass}}{{=XML(b', '.join([A(f,_href=URL(a=app,c=c[:-3],f=f)).xml() for f in functions[c]]))}}
 			</span>
 		</li>
 		{{pass}}
@@ -145,7 +145,7 @@ for c in controllers: controller_functions+=[c[:-3]+'/%s.html'%x for x in functi
 			</span>
 			<span class="extras celled">
 				{{if c in extend:}}{{=T("extends")}} <b>{{=extend[c]}}</b> {{pass}}
-				{{if include[c]:}}{{=T("includes")}} {{pass}}{{=XML(', '.join([B(f).xml() for f in include[c]]))}}
+				{{if include[c]:}}{{=T("includes")}} {{pass}}{{=XML(b', '.join([B(f).xml() for f in include[c]]))}}
 			</span>
 		</li>
 		{{pass}}

applications/examples/views/default/download.html

@@ -62,16 +62,16 @@
 </center>
 
 <p style="text-align:left;">
-  The source code version works on all supported platforms, including Linux, but it requires Python 2.6, or 2.7 (recommended).
+  The source code version works on Windows and most Unix systems, including <b>Linux</b>,  <b>BSD</b> and  <b>Mac</b> . It requires Python 2.6 (no more supported),  Python 2.7 (stable) or Python 3.5+ (recommended for new projects) already installed on your system.
-  It runs on Windows and most Unix systems, including <b>Linux</b> and <b>BSD</b>.
+  There are also binary packages for Windows and Mac OS X. They include the Python 2.7 interpreter so you do not need to have it pre-installed.
 </p>
 
 <h3>Instructions</h3>
-<p>After download, unzip it and click on web2py.exe (windows) or web2py.app (osx).
+<p>With the binary packages, after download, just unzip it and then click on web2py.exe (windows) or web2py.app (osx).
-  To run from source, type:</p>
+  If you prefer to run it from source with your own Python interpreter alreay installed, type:</p>
-{{=CODE("python2.7 web2py.py", language=None, counter='>', _class='boxCode')}}
+{{=CODE("python web2py.py", language=None, counter='>', _class='boxCode')}}
 <p>or for more info type:</p>
-{{=CODE("python2.7 web2py.py -h", language=None, counter='>', _class='boxCode')}}
+{{=CODE("python web2py.py -h", language=None, counter='>', _class='boxCode')}}
 
 
 <h3>Caveats</h3>

applications/examples/views/default/index.html

@@ -4,7 +4,7 @@
   <div class="twothirds">
     <div class="padded">
       <h3><img src="{{=URL('static/images', 'web2py_logo.png')}}"> Web Framework</h3>
-      <p>Free open source full-stack framework for rapid development of fast, scalable, <a href="http://www.web2py.com/book/default/chapter/01#Security" target="_blank">secure</a> and portable database-driven web-based applications. Written and programmable in <a href="http://www.python.org" target="_blank">Python</a>.</p>
+      <p>Free open source full-stack framework for rapid development of fast, scalable, <a href="http://www.web2py.com/book/default/chapter/01#Security" target="_blank">secure</a> and portable database-driven web-based applications. Written and programmable in <a href="http://www.python.org" target="_blank">Python</a> (version 3 and 2.7).</p>
       <table width="100%">
 	<tr>
 	  <td>
@@ -18,7 +18,7 @@
 	    </a>
 	  </td>
 	  <td>
-	    <a class="noeffect" href="http://link.packtpub.com/SUlnrN">
+	    <a class="noeffect" href="https://www.packtpub.com/web-development/web2py-application-development-cookbook">
 	      <img src="{{=URL('static','images/book-recipes.png')}}" />
 	    </a>
 	  </td>

applications/welcome/static/css/web2py-bootstrap4.css

@@ -348,3 +348,13 @@ td.w2p_fc,
 .icon.pen:before { content: "\f040";}
 .icon.arrowright:before { content: "\f061";}
 .icon.magnifier:before { content: "\f002";}
+
+.web2py_table_selectable_actions {
+   padding-top: 10px;
+   float: right;
+}
+
+.web2py_table_selectable_actions input {
+   padding: 5px 7px;
+   margin-right: 10px;
+}

applications/welcome/static/js/web2py-bootstrap4.js

@@ -52,7 +52,7 @@
         });
       }
       var ul = this;
-      $(ul).find(":text").addClass('form-control').wrap("<div class='input-group'></div>").after('<div class="input-group-addon"><i class="glyphicon glyphicon-plus"></i></div><div class="input-group-addon"><i class="glyphicon glyphicon-minus"></i></div>').keypress(function(e) {
+      $(ul).find(":text").addClass('form-control').wrap("<div class='input-group'></div>").after('<div class="input-group-append"><i class="fa fa-plus-circle"></i></div>&nbsp;<div class="input-group-append"><i class="fa fa-minus-circle"></i></div>').keypress(function(e) {
         return (e.which == 13) ? pe(ul, e) : true;
       }).next().click(function(e) {
         pe(ul, e);

applications/welcome/views/default/index.html

@@ -1,7 +1,7 @@
 {{extend 'layout.html'}}
 
 {{block header}}
-<div class="jumbotron jumbotron-fluid" style="background-color: #333; color:white; padding:30px;word-wrap:break-word;">
+<div class="jumbotron jumbotron-fluid background" style="background-color: #333; color:white; padding:30px;word-wrap:break-word;">
   <div class="container center">
     <h1 class="display-5">/{{=request.application}}/{{=request.controller}}/{{=request.function}}</h1>
   </div>

applications/welcome/views/generic.csv

@@ -0,0 +1,17 @@
+{{"""Usage:
+
+  def controller():
+      return {"": db().select(db.thing.ALL)}
+
+And then visit that controller with a .csv extention name
+"""
+}}{{if len(response._vars)==1:}}{{
+# Not yet find a Python 2/3 compatible StringIO pattern,
+# we avoid this solution http://web2py.com/books/default/chapter/29/10/services#CSV
+# Here we buffer the entire csv file instead (it is your controller's job to limit the volume anyway),
+# based on: http://web2py.com/books/default/chapter/29/06/the-database-abstraction-layer#CSV-one-Table-at-a-time-
+
+content = response._vars[next(iter(response._vars))]
+response.headers['Content-Type'] = 'application/vnd.ms-excel'
+response.write(str(content), escape=False)
+}}{{pass}}

applications/welcome/views/layout.html

@@ -35,7 +35,7 @@
   <body>
     <div class="w2p_flash alert alert-dismissable">{{=response.flash or ''}}</div>
     <!-- Navbar ======================================= -->
-    <nav class="navbar navbar-light navbar-expand-md bg-faded justify-content-center">
+    <nav class="navbar navbar-light navbar-expand-md bg-faded bg-dark navbar-dark justify-content-center">
        <a href="http://web2py.com" class="navbar-brand d-flex w-50 mr-auto">web2py</a>
        <button class="navbar-toggler" type="button" data-toggle="collapse" data-target="#navbarNavDropdown" aria-controls="navbarNavDropdown" aria-expanded="false" aria-label="Toggle navigation">
          <span class="navbar-toggler-icon"></span>

gluon/__init__.py

@@ -31,6 +31,7 @@ except ImportError:
         "You can also download a complete copy from http://www.web2py.com."
     )
 
+
 from .globals import current
 from .html import *
 from .validators import *

gluon/_compat.py

@@ -1,163 +1,7 @@
-import sys
+from pydal._compat import *
-import hashlib
-import os
-
-PY2 = sys.version_info[0] == 2
-
-_identity = lambda x: x
 
 if PY2:
-    import cPickle as pickle
-    from cStringIO import StringIO
-    import copy_reg as copyreg
-    from HTMLParser import HTMLParser
-    import urlparse
-    from htmlentitydefs import entitydefs, name2codepoint
-    import __builtin__ as builtin
-    import thread
-    import Cookie
-    import urllib2
-    import Queue
-    import ConfigParser as configparser
-    from email.MIMEBase import MIMEBase
-    from email.Header import Header
-    from email import Encoders, Charset
-    from email.MIMEMultipart import MIMEMultipart
-    from email.MIMEText import MIMEText
-    from email.Charset import add_charset, QP as charset_QP
-    from urllib import FancyURLopener, urlencode, urlopen
-    from urllib import quote as urllib_quote, unquote as urllib_unquote, quote_plus as urllib_quote_plus
-    from string import maketrans
-    from types import ClassType
-    import cgi
-    import cookielib
-    from xmlrpclib import ProtocolError
     from gluon.contrib import ipaddress
-    BytesIO = StringIO
-    reduce = reduce
-    reload = reload
-    hashlib_md5 = hashlib.md5
-    iterkeys = lambda d: d.iterkeys()
-    itervalues = lambda d: d.itervalues()
-    iteritems = lambda d: d.iteritems()
-    integer_types = (int, long)
-    string_types = (str, unicode)
-    text_type = unicode
-    basestring = basestring
-    xrange = xrange
-    long = long
-    unichr = unichr
-    unicodeT = unicode
-
-    def implements_bool(cls):
-        cls.__nonzero__ = cls.__bool__
-        del cls.__bool__
-        return cls
-
-    def implements_iterator(cls):
-        cls.next = cls.__next__
-        del cls.__next__
-        return cls
-
-    def to_bytes(obj, charset='utf-8', errors='strict'):
-        if obj is None:
-            return None
-        if isinstance(obj, (bytes, bytearray, buffer)):
-            return bytes(obj)
-        if hasattr(obj, 'encode'):
-            return obj.encode(charset, errors)
-        raise TypeError('Expected bytes')
-
-    def to_native(obj, charset='utf8', errors='strict'):
-        if obj is None or isinstance(obj, str):
-            return obj
-        return obj.encode(charset, errors)
-
-
 else:
-    import pickle
-    from io import StringIO, BytesIO
-    import copyreg
-    from importlib import reload
-    from functools import reduce
-    from html.parser import HTMLParser
-    from http import cookies as Cookie
-    from urllib import parse as urlparse
-    from urllib import request as urllib2
-    from html.entities import entitydefs, name2codepoint
-    import builtins as builtin
-    import _thread as thread
-    import configparser
-    import queue as Queue
-    from email.mime.base import MIMEBase
-    from email.mime.multipart import MIMEMultipart
-    from email.mime.text import MIMEText
-    from email import encoders as Encoders
-    from email.header import Header
-    from email.charset import Charset, add_charset, QP as charset_QP
-    from urllib.request import FancyURLopener, urlopen
-    from urllib.parse import quote as urllib_quote, unquote as urllib_unquote, urlencode, quote_plus as urllib_quote_plus
-    from http import cookiejar as cookielib
-    from xmlrpc.client import ProtocolError
-    import html # warning, this is the python3 module and not the web2py html module
     import ipaddress
-    hashlib_md5 = lambda s: hashlib.md5(bytes(s, 'utf8'))
-    iterkeys = lambda d: iter(d.keys())
-    itervalues = lambda d: iter(d.values())
-    iteritems = lambda d: iter(d.items())
-    integer_types = (int,)
-    string_types = (str,)
-    text_type = str
-    basestring = str
-    xrange = range
-    long = int
-    unichr = chr
-    unicodeT = str
-    maketrans = str.maketrans
-    ClassType = type
     
-    implements_iterator = _identity
-    implements_bool = _identity
-
-    def to_bytes(obj, charset='utf-8', errors='strict'):
-        if obj is None:
-            return None
-        if isinstance(obj, (bytes, bytearray, memoryview)):
-            return bytes(obj)
-        if hasattr(obj, 'encode'):
-            return obj.encode(charset, errors)
-        raise TypeError('Expected bytes')
-
-    def to_native(obj, charset='utf8', errors='strict'):
-        if obj is None or isinstance(obj, str):
-            return obj
-        return obj.decode(charset, errors)
-
-
-def with_metaclass(meta, *bases):
-    """Create a base class with a metaclass."""
-    # This requires a bit of explanation: the basic idea is to make a dummy
-    # metaclass for one level of class instantiation that replaces itself with
-    # the actual metaclass.
-    class metaclass(meta):
-        __call__ = type.__call__
-        __init__ = type.__init__
-
-        def __new__(cls, name, this_bases, d):
-            if this_bases is None:
-                return type.__new__(cls, name, (), d)
-            return meta(name, bases, d)
-    return metaclass('temporary_class', None, {})
-
-
-def to_unicode(obj, charset='utf-8', errors='strict'):
-    if obj is None:
-        return None
-    if not hasattr(obj, 'decode'):
-        return text_type(obj)
-    return obj.decode(charset, errors)
-
-
-# shortcuts
-pjoin = os.path.join
-exists = os.path.exists

gluon/compileapp.py

@@ -15,7 +15,7 @@ Note:
 
 import re
 import fnmatch
-import os
+import os, sys
 import copy
 import random
 from gluon._compat import builtin, PY2, unicodeT, to_native, to_bytes, iteritems, basestring, reduce, xrange, long, reload
@@ -52,7 +52,10 @@ is_gae = settings.global_settings.web2py_runtime_gae
 is_jython = settings.global_settings.is_jython
 pjoin = os.path.join
 
-marshal_header_size = 8 if PY2 else 12
+if PY2:
+    marshal_header_size = 8
+else:
+    marshal_header_size = 16 if sys.version_info[1] >= 7 else 12
 
 TEST_CODE = \
     r"""

gluon/contrib/redis_cache.py

@@ -22,7 +22,7 @@ logger = logging.getLogger("web2py.cache.redis")
 locker = Lock()
 
 
-def RedisCache(redis_conn=None, debug=False, with_lock=False, fail_gracefully=False, db=None):
+def RedisCache(redis_conn=None, debug=False, with_lock=False, fail_gracefully=False, db=None, application=None):
     """
     Usage example: put in models::
 
@@ -47,6 +47,8 @@ def RedisCache(redis_conn=None, debug=False, with_lock=False, fail_gracefully=Fa
             When True, only one thread/process can set a value concurrently
         fail_gracefully: if redis is unavailable, returns the value computing it
             instead of raising an exception
+        application: if provided, it is used to construct the instance_name,
+            allowing to share cache between different applications if needed
 
     It can be used pretty much the same as cache.ram()
     When you use cache.redis directly you can use :
@@ -83,11 +85,14 @@ def RedisCache(redis_conn=None, debug=False, with_lock=False, fail_gracefully=Fa
 
     locker.acquire()
     try:
-        instance_name = 'redis_instance_' + current.request.application
+        if not application:
+            application = current.request.application
+        instance_name = 'redis_instance_' + application
         if not hasattr(RedisCache, instance_name):
             setattr(RedisCache, instance_name,
                     RedisClient(redis_conn=redis_conn, debug=debug,
-                                with_lock=with_lock, fail_gracefully=fail_gracefully))
+                                with_lock=with_lock, fail_gracefully=fail_gracefully,
+                                application=application))
         return getattr(RedisCache, instance_name)
     finally:
         locker.release()
@@ -100,14 +105,15 @@ class RedisClient(object):
     RETRIES = 0
 
     def __init__(self, redis_conn=None, debug=False,
-                 with_lock=False, fail_gracefully=False):
+                 with_lock=False, fail_gracefully=False, application=None):
         self.request = current.request
         self.debug = debug
         self.with_lock = with_lock
         self.fail_gracefully = fail_gracefully
-        self.prefix = "w2p:cache:%s:" % self.request.application
+        self.application = application
+        self.prefix = "w2p:cache:%s:" % application
         if self.request:
-            app = self.request.application
+            app = application
         else:
             app = ''
 
@@ -120,7 +126,7 @@ class RedisClient(object):
         else:
             self.storage = self.meta_storage[app]
 
-        self.cache_set_key = 'w2p:%s:___cache_set' % self.request.application
+        self.cache_set_key = 'w2p:%s:___cache_set' % application
 
         self.r_server = redis_conn
         self._release_script = register_release_lock(self.r_server)
@@ -180,7 +186,7 @@ class RedisClient(object):
             self.r_server.incr('web2py_cache_statistics:misses')
         cache_set_key = self.cache_set_key
         expire_at = int(time.time() + time_expire) + 120
-        bucket_key = "%s:%s" % (cache_set_key, expire_at / 60)
+        bucket_key = "%s:%s" % (cache_set_key, expire_at // 60)
         value = f()
         value_ = pickle.dumps(value, pickle.HIGHEST_PROTOCOL)
         if time_expire == 0:
@@ -196,7 +202,7 @@ class RedisClient(object):
         # add the key to the bucket
         p.sadd(bucket_key, key)
         # expire the bucket properly
-        p.expireat(bucket_key, ((expire_at / 60) + 1) * 60)
+        p.expireat(bucket_key, ((expire_at // 60) + 1) * 60)
         p.execute()
         return value
 
@@ -276,8 +282,7 @@ class RedisClient(object):
             )
         stats_collector['w2p_keys'] = dict()
 
-        for a in self.r_server.keys("w2p:%s:*" % (
+        for a in self.r_server.keys("w2p:%s:*" % self.application):
-                self.request.application)):
             stats_collector['w2p_keys']["%s_expire_in_sec" % a] = self.r_server.ttl(a)
         return stats_collector
 

gluon/contrib/redis_scheduler.py

@@ -487,7 +487,7 @@ class RScheduler(Scheduler):
             # calc next_run_time based on available slots
             # see #1191
             next_run_time = task.start_time
-            secondspassed = self.total_seconds(now - next_run_time)
+            secondspassed = (now - next_run_time).total_seconds()
             steps = secondspassed // task.period + 1
             next_run_time += datetime.timedelta(seconds=task.period * steps)
 

gluon/contrib/redis_session.py

@@ -186,8 +186,8 @@ class MockQuery(object):
             if rtn:
                 if self.unique_key:
                     # make sure the id and unique_key are correct
-                    if rtn[b'unique_key'] == to_bytes(self.unique_key):
+                    if rtn['unique_key'] == self.unique_key:
-                        rtn[b'update_record'] = self.update  # update record support
+                        rtn['update_record'] = self.update  # update record support
                     else:
                         rtn = None
             return [Storage(rtn)] if rtn else []

gluon/contrib/simplejsonrpc.py

@@ -35,6 +35,8 @@ import json
 class JSONRPCError(RuntimeError):
     "Error object for remote procedure call fail"
     def __init__(self, code, message, data=''):        
+        if isinstance(data, basestring):
+            data = [data]
         value = "%s: %s\n%s" % (code, message, '\n'.join(data))
         RuntimeError.__init__(self, value)
         self.code = code
@@ -82,13 +84,14 @@ class JSONSafeTransport(JSONTransportMixin, SafeTransport):
 class ServerProxy(object):
     "JSON RPC Simple Client Service Proxy"
 
-    def __init__(self, uri, transport=None, encoding=None, verbose=0,version=None):
+    def __init__(self, uri, transport=None, encoding=None, verbose=0, version=None, json_encoder=None):
         self.location = uri             # server location (url)
         self.trace = verbose            # show debug messages
         self.exceptions = True          # raise errors? (JSONRPCError)
         self.timeout = None
         self.json_request = self.json_response = ''
         self.version = version          # '2.0' for jsonrpc2
+        self.json_encoder = json_encoder  # Allow for a custom JSON encoding class
 
         type, uri = urllib.splittype(uri)
         if type not in ("http", "https"):
@@ -116,7 +119,7 @@ class ServerProxy(object):
         data = {'id': request_id, 'method': method, 'params': args or vars, }
         if self.version:
             data['jsonrpc'] = self.version #mandatory key/value for jsonrpc2 validation else err -32600
-        request = json.dumps(data)
+        request = json.dumps(data, cls=self.json_encoder)
 
         # make HTTP request (retry if connection is lost)
         response = self.__transport.request(

gluon/globals.py

@@ -969,7 +969,7 @@ class Session(Storage):
                     if row:
                         # rows[0].update_record(locked=True)
                         # Unpickle the data
-                        session_data = pickle.loads(row[b'session_data'])
+                        session_data = pickle.loads(row['session_data'])
                         self.update(session_data)
                         response.session_new = False
                     else:
@@ -1051,7 +1051,7 @@ class Session(Storage):
             if record_id.isdigit() and long(record_id) > 0:
                 new_unique_key = web2py_uuid()
                 row = table(record_id)
-                if row and row[b'unique_key'] == to_bytes(unique_key):
+                if row and row['unique_key'] == unique_key:
                     table._db(table.id == record_id).update(unique_key=new_unique_key)
                 else:
                     record_id = None
@@ -1075,6 +1075,16 @@ class Session(Storage):
             scookies['HttpOnly'] = True
         if self._secure:
             scookies['secure'] = True
+        if self._same_site is None:
+            # Using SameSite Lax Mode is the default
+            # You actually have to call session.samesite(False) if you really
+            # dont want the extra protection provided by the SameSite header 
+            self._same_site = 'Lax'
+        if self._same_site:
+            if 'samesite' not in Cookie.Morsel._reserved:
+                # Python version 3.7 and lower needs this
+                Cookie.Morsel._reserved['samesite'] = 'SameSite'
+            scookies['samesite'] = self._same_site
 
     def clear_session_cookies(self):
         request = current.request
@@ -1153,6 +1163,9 @@ class Session(Storage):
     def secure(self):
         self._secure = True
 
+    def samesite(self, mode='Lax'):
+        self._same_site = mode
+
     def forget(self, response=None):
         self._close(response)
         self._forget = True
@@ -1180,7 +1193,7 @@ class Session(Storage):
 
     def _unchanged(self, response):
         if response.session_new:
-            internal = ['_last_timestamp', '_secure', '_start_timestamp']
+            internal = ['_last_timestamp', '_secure', '_start_timestamp', '_same_site']
             for item in self.keys():
                 if item not in internal:
                     return False

gluon/languages.py

@@ -311,7 +311,7 @@ def write_plural_dict(filename, contents):
     try:
         fp = LockedFile(filename, 'w')
         fp.write('#!/usr/bin/env python\n# -*- coding: utf-8 -*-\n{\n# "singular form (0)": ["first plural form (1)", "second plural form (2)", ...],\n')
-        for key in sorted(contents, sort_function):
+        for key in sorted(contents, key=sort_function):
             forms = '[' + ','.join([repr(Utf8(form))
                                     for form in contents[key]]) + ']'
             fp.write('%s: %s,\n' % (repr(Utf8(key)), forms))
@@ -325,8 +325,8 @@ def write_plural_dict(filename, contents):
             fp.close()
 
 
-def sort_function(x, y):
+def sort_function(x):
-    return cmp(unicode(x, 'utf-8').lower(), unicode(y, 'utf-8').lower())
+    return unicode(x, 'utf-8').lower()
 
 
 def write_dict(filename, contents):
@@ -936,8 +936,8 @@ class translator(object):
                     word = w[1:]
                     fun = cap_fun
                 if i is not None:
-                    return fun(self.plural(word, symbols[int(i)]))
+                    return to_native(fun(self.plural(word, symbols[int(i)])))
-                return fun(word)
+                return to_native(fun(word))
 
             def sub_dict(m):
                 """ word(key or num)

gluon/main.py

@@ -31,7 +31,7 @@ from gluon._compat import Cookie, urllib2
 
 from gluon.fileutils import abspath, write_file
 from gluon.settings import global_settings
-from gluon.utils import web2py_uuid
+from gluon.utils import web2py_uuid, unlocalised_http_header_date
 from gluon.admin import add_path_first, create_missing_folders, create_missing_app_folders
 from gluon.globals import current
 
@@ -199,8 +199,7 @@ def serve_controller(request, response, session):
         ('Content-Type', contenttype('.' + request.extension)),
         ('Cache-Control',
          'no-store, no-cache, must-revalidate, post-check=0, pre-check=0'),
-        ('Expires', time.strftime('%a, %d %b %Y %H:%M:%S GMT',
+        ('Expires', unlocalised_http_header_date(time.gmtime())),
-                                  time.gmtime())),
         ('Pragma', 'no-cache')]
     for key, value in default_headers:
         response.headers.setdefault(key, value)

gluon/packages/template/__init__.py

@@ -0,0 +1 @@
+

gluon/shell.py

@@ -37,7 +37,7 @@ logger = logging.getLogger("web2py")
 
 if not PY2:
     def execfile(filename, global_vars=None, local_vars=None):
-        with open(filename) as f:
+        with open(filename, "rb") as f:
             code = compile(f.read(), filename, 'exec')
             exec(code, global_vars, local_vars)
     raw_input = input

gluon/sqlhtml.py

@@ -699,18 +699,20 @@ class AutocompleteWidget(object):
             if isinstance(field, Field.Virtual):
                 records = []
                 table_rows = self.db(self.db[field.tablename]).select(orderby=self.orderby)
-                count = 0
+                count = self.limitby[1] if self.limitby else -1
                 for row in table_rows:
                     if self.at_beginning:
                         if row[field.name].lower().startswith(kword):
-                            count += 1
+                            count -= 1
                             records.append(row)
                     else:
                         if kword in row[field.name].lower():
-                            count += 1
+                            count -= 1
                             records.append(row)
-                    if count == 10:
+                    if count == 0:
                         break
+                if self.limitby and self.limitby[0]:
+                    records = records[self.limitby[0]:]
                 rows = Rows(self.db, records, table_rows.colnames,
                             compact=table_rows.compact)
             elif settings and settings.global_settings.web2py_runtime_gae:
@@ -800,8 +802,8 @@ class AutocompleteWidget(object):
                     $('#%(key3)s').val('');
                     var e=e_.which?e_.which:e_.keyCode;
                     function %(u)s(){
-                        $('#%(id)s').val($('#%(key)s :selected').text());
+                        $('#%(id)s').val($('#%(key)s option:selected').text());
-                        $('#%(key3)s').val($('#%(key)s').val())
+                        $('#%(key3)s').val($('#%(key)s option:selected').val())
                     };
                     if(e==39) %(u)s();
                     else if(e==40) {
@@ -822,7 +824,7 @@ class AutocompleteWidget(object):
                                     $('#%(id)s').next('.error').hide();
                                     $('#%(div_id)s').html(data).show().focus();
                                     $('#%(div_id)s select').css('width',$('#%(id)s').css('width'));
-                                    $('#%(key3)s').val($('#%(key)s').val());
+                                    $('#%(key3)s').val($('#%(key)s option:selected').val());
                                     $('#%(key)s').change(%(u)s).click(%(u)s);
                                 };
                             });
@@ -853,7 +855,7 @@ class AutocompleteWidget(object):
             function doit(e_) {
                 var e=e_.which?e_.which:e_.keyCode;
                 function %(u)s(){
-                    $('#%(id)s').val($('#%(key)s').val())
+                    $('#%(id)s').val($('#%(key)s option:selected').val())
                 };
                 if(e==39) %(u)s();
                 else if(e==40) {
@@ -1090,7 +1092,7 @@ def formstyle_bootstrap3_inline_factory(col_label_size=3):
 
 # bootstrap 4
 def formstyle_bootstrap4_stacked(form, fields):
-    """ bootstrap 3 format form layout
+    """ bootstrap 4 format form layout
 
     Note:
         Experimental!
@@ -1139,7 +1141,7 @@ def formstyle_bootstrap4_stacked(form, fields):
 
 
 def formstyle_bootstrap4_inline_factory(col_label_size=3):
-    """ bootstrap 3 horizontal form layout
+    """ bootstrap 4 horizontal form layout
 
     Note:
         Experimental!
@@ -1349,7 +1351,7 @@ class SQLFORM(FORM):
             if not readonly:
                 if not record:
                     # create form should only show writable fields
-                    fields = [f.name for f in table if (ignore_rw or f.writable) and not f.compute]
+                    fields = [f.name for f in table if (ignore_rw or f.writable or (f.readable and f.default)) and not f.compute]
                 else:
                     # update form should also show readable fields and computed fields (but in reaodnly mode)
                     fields = [f.name for f in table if (ignore_rw or f.writable or f.readable)]
@@ -1910,23 +1912,10 @@ class SQLFORM(FORM):
                     fields[fieldname] = [safe_int(
                         x) for x in (value and [value] or [])]
             elif field.type == 'integer':
-                if value is not None:
+                fields[fieldname] = safe_int(value, None)
-                    fields[fieldname] = safe_int(value)
             elif field.type.startswith('reference'):
-                ## Avoid "constraint violation" exception when you have a
-                ## optional reference field without the dropdown in form. I.e.,
-                ## a field with code to be typed, in a data-entry form.
-                ##
-                ## When your reference field doesn't have IS_EMPTY_OR()
-                ## validator, "value" will come here as a string. So,
-                ## safe_int() will return 0. In this case, insert will raise
-                ## the constraint violation because there's no id=0 in
-                ## referenced table.
                 if isinstance(self.table, Table) and not keyed:
-                    if not value:
+                    fields[fieldname] = safe_int(value, None)
-                        fields[fieldname] = None
-                    else:
-                        fields[fieldname] = safe_int(value)
             elif field.type == 'double':
                 if value is not None:
                     fields[fieldname] = safe_float(value)
@@ -2021,7 +2010,7 @@ class SQLFORM(FORM):
         to hold the fields.
         """
         # this is here to avoid circular references
-        from gluon.dal import DAL
+        from gluon.dal import DAL, _default_validators
         # Define a table name, this way it can be logical to our CSS.
         # And if you switch from using SQLFORM to SQLFORM.factory
         # your same css definitions will still apply.
@@ -2034,8 +2023,9 @@ class SQLFORM(FORM):
 
         # Clone fields, while passing tables straight through
         fields_with_clones = [f.clone() if isinstance(f, Field) else f for f in fields]
-
+        dummy_dal = DAL(None)
-        return SQLFORM(DAL(None).define_table(table_name, *fields_with_clones), **attributes)
+        dummy_dal.validators_method = lambda f: _default_validators(dummy_dal, f) # See https://github.com/web2py/web2py/issues/2007
+        return SQLFORM(dummy_dal.define_table(table_name, *fields_with_clones), **attributes)
 
     @staticmethod
     def build_query(fields, keywords):
@@ -2312,7 +2302,7 @@ class SQLFORM(FORM):
                       button='button btn btn-default btn-secondary',
                       buttontext='buttontext button',
                       buttonadd='icon plus icon-plus glyphicon glyphicon-plus',
-                      buttonback='icon leftarrow icon-arrow-left glyphicon glyphicon-arrow-left',
+                      buttonback='icon arrowleft icon-arrow-left glyphicon glyphicon-arrow-left',
                       buttonexport='icon downarrow icon-download glyphicon glyphicon-download',
                       buttondelete='icon trash icon-trash glyphicon glyphicon-trash',
                       buttonedit='icon pen icon-pencil glyphicon glyphicon-pencil',
@@ -2690,13 +2680,13 @@ class SQLFORM(FORM):
                             dbset = dbset(SQLFORM.build_query(
                                 sfields, keywords))
                         rows = dbset.select(left=left, orderby=orderby,
-                                            cacheable=True, *selectable_columns)
+                                            cacheable=True, *expcolumns)
                     except Exception as e:
                         response.flash = T('Internal Error')
                         rows = []
                 else:
                     rows = dbset.select(left=left, orderby=orderby,
-                                        cacheable=True, *selectable_columns)
+                                        cacheable=True, *expcolumns)
 
                 value = exportManager[export_type]
                 clazz = value[0] if hasattr(value, '__getitem__') else value
@@ -2855,9 +2845,9 @@ class SQLFORM(FORM):
         if paginate and dbset._db._adapter.dbengine == 'google:datastore' and use_cursor:
             cursor = request.vars.cursor or True
             limitby = (0, paginate)
-            page = safe_int(request.vars.page, 1) - 1
+            page = safe_int(request.vars.page or 1, 1) - 1
         elif paginate and paginate < nrows:
-            page = safe_int(request.vars.page, 1) - 1
+            page = safe_int(request.vars.page or 1, 1) - 1
             limitby = (paginate * page, paginate * (page + 1))
         else:
             limitby = None
@@ -2899,7 +2889,7 @@ class SQLFORM(FORM):
         paginator = UL()
         if paginate and dbset._db._adapter.dbengine == 'google:datastore' and use_cursor:
             # this means we may have a large table with an unknown number of rows.
-            page = safe_int(request.vars.page, 1) - 1
+            page = safe_int(request.vars.page or 1, 1) - 1
             paginator.append(LI('page %s' % (page + 1)))
             if next_cursor:
                 d = dict(page=page + 2, cursor=next_cursor)
@@ -2918,7 +2908,7 @@ class SQLFORM(FORM):
             npages, reminder = divmod(nrows, paginate)
             if reminder:
                 npages += 1
-            page = safe_int(request.vars.page, 1) - 1
+            page = safe_int(request.vars.page or 1, 1) - 1
 
             def self_link(name, p):
                 d = dict(page=p + 1)
@@ -3089,8 +3079,9 @@ class SQLFORM(FORM):
 
                 if formstyle == 'bootstrap':
                     # add space between buttons
-                    # inputs = sum([[inp, ' '] for inp in inputs], [])[:-1]
                     htmltable = FORM(htmltable, DIV(_class='form-actions', *inputs))
+                elif not callable(formstyle) and 'bootstrap' in formstyle: # Same for bootstrap 3 & 4
+                     htmltable = FORM(htmltable, DIV(_class='form-group web2py_table_selectable_actions', *inputs))
                 else:
                     htmltable = FORM(htmltable, *inputs)
 

gluon/streamer.py

@@ -16,6 +16,7 @@ import time
 import re
 import errno
 from gluon.http import HTTP
+from gluon.utils import unlocalised_http_header_date
 from gluon.contenttype import contenttype
 from gluon._compat import PY2
 
@@ -74,7 +75,7 @@ def stream_file_or_304_or_206(
     stat_file = os.stat(static_file)
     fsize = stat_file[stat.ST_SIZE]
     modified = stat_file[stat.ST_MTIME]
-    mtime = time.strftime('%a, %d %b %Y %H:%M:%S GMT', time.gmtime(modified))
+    mtime = unlocalised_http_header_date(time.gmtime(modified))
     headers.setdefault('Content-Type', contenttype(static_file))
     headers.setdefault('Last-Modified', mtime)
     headers.setdefault('Pragma', 'cache')

gluon/template.py

@@ -1,938 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-"""
-| This file is part of the web2py Web Framework
-| License: LGPLv3 (http://www.gnu.org/licenses/lgpl.html)
-| Author: Thadeus Burgess
-| Contributors:
-| - Massimo Di Pierro for creating the original gluon/template.py
-| - Jonathan Lundell for extensively testing the regex on Jython.
-| - Limodou (creater of uliweb) who inspired the block-element support for web2py.
-
-Templating syntax
-------------------
-"""
-
-import os
-import cgi
-import logging
-from re import compile, sub, escape, DOTALL
-from gluon._compat import StringIO, unicodeT, to_unicode, to_bytes, to_native
-
-try:
-    # have web2py
-    from gluon.restricted import RestrictedError
-    from gluon.globals import current
-except ImportError:
-    # do not have web2py
-    current = None
-
-    def RestrictedError(a, b, c):
-        logging.error(str(a) + ':' + str(b) + ':' + str(c))
-        return RuntimeError
-
-
-class Node(object):
-    """
-    Basic Container Object
-    """
-    def __init__(self, value=None, pre_extend=False):
-        self.value = value
-        self.pre_extend = pre_extend
-
-    def __str__(self):
-        return str(self.value)
-
-
-class SuperNode(Node):
-    def __init__(self, name='', pre_extend=False):
-        self.name = name
-        self.value = None
-        self.pre_extend = pre_extend
-
-    def __str__(self):
-        if self.value:
-            return str(self.value)
-        else:
-            # raise SyntaxError("Undefined parent block ``%s``. \n" % self.name + "You must define a block before referencing it.\nMake sure you have not left out an ``{{end}}`` tag." )
-            return ''
-
-    def __repr__(self):
-        return "%s->%s" % (self.name, self.value)
-
-
-def output_aux(node, blocks):
-    # If we have a block level
-    #   If we can override this block.
-    #     Override block from vars.
-    #   Else we take the default
-    # Else its just a string
-    return (blocks[node.name].output(blocks)
-            if node.name in blocks else
-            node.output(blocks)) \
-        if isinstance(node, BlockNode) \
-        else str(node)
-
-
-class BlockNode(Node):
-    """
-    Block Container.
-
-    This Node can contain other Nodes and will render in a hierarchical order
-    of when nodes were added.
-
-    ie::
-
-        {{ block test }}
-            This is default block test
-        {{ end }}
-
-    """
-    def __init__(self, name='', pre_extend=False, delimiters=('{{', '}}')):
-        """
-        name - Name of this Node.
-        """
-        self.nodes = []
-        self.name = name
-        self.pre_extend = pre_extend
-        self.left, self.right = delimiters
-
-    def __repr__(self):
-        lines = ['%sblock %s%s' % (self.left, self.name, self.right)]
-        lines += [str(node) for node in self.nodes]
-        lines.append('%send%s' % (self.left, self.right))
-        return ''.join(lines)
-
-    def __str__(self):
-        """
-        Get this BlockNodes content, not including child Nodes
-        """
-        return ''.join(str(node) for node in self.nodes
-                       if not isinstance(node, BlockNode))
-
-    def append(self, node):
-        """
-        Adds an element to the nodes.
-
-        Args:
-            node: Node object or string to append.
-        """
-        if isinstance(node, str) or isinstance(node, Node):
-            self.nodes.append(node)
-        else:
-            raise TypeError("Invalid type; must be instance of ``str`` or ``BlockNode``. %s" % node)
-
-    def extend(self, other):
-        """
-        Extends the list of nodes with another BlockNode class.
-
-        Args:
-            other: BlockNode or Content object to extend from.
-        """
-        if isinstance(other, BlockNode):
-            self.nodes.extend(other.nodes)
-        else:
-            raise TypeError(
-                "Invalid type; must be instance of ``BlockNode``. %s" % other)
-
-    def output(self, blocks):
-        """
-        Merges all nodes into a single string.
-
-        Args:
-            blocks: Dictionary of blocks that are extending from this template.
-        """
-        return ''.join(output_aux(node, blocks) for node in self.nodes)
-
-
-class Content(BlockNode):
-    """
-    Parent Container -- Used as the root level BlockNode.
-
-    Contains functions that operate as such.
-
-    Args:
-        name: Unique name for this BlockNode
-    """
-    def __init__(self, name="ContentBlock", pre_extend=False):
-        self.name = name
-        self.nodes = []
-        self.blocks = {}
-        self.pre_extend = pre_extend
-
-    def __str__(self):
-        return ''.join(output_aux(node, self.blocks) for node in self.nodes)
-
-    def _insert(self, other, index=0):
-        """
-        Inserts object at index.
-        """
-        if isinstance(other, (str, Node)):
-            self.nodes.insert(index, other)
-        else:
-            raise TypeError(
-                "Invalid type, must be instance of ``str`` or ``Node``.")
-
-    def insert(self, other, index=0):
-        """
-        Inserts object at index.
-
-        You may pass a list of objects and have them inserted.
-        """
-        if isinstance(other, (list, tuple)):
-            # Must reverse so the order stays the same.
-            other.reverse()
-            for item in other:
-                self._insert(item, index)
-        else:
-            self._insert(other, index)
-
-    def append(self, node):
-        """
-        Adds a node to list. If it is a BlockNode then we assign a block for it.
-        """
-        if isinstance(node, (str, Node)):
-            self.nodes.append(node)
-            if isinstance(node, BlockNode):
-                self.blocks[node.name] = node
-        else:
-            raise TypeError("Invalid type, must be instance of ``str`` or ``BlockNode``. %s" % node)
-
-    def extend(self, other):
-        """
-        Extends the objects list of nodes with another objects nodes
-        """
-        if isinstance(other, BlockNode):
-            self.nodes.extend(other.nodes)
-            self.blocks.update(other.blocks)
-        else:
-            raise TypeError(
-                "Invalid type; must be instance of ``BlockNode``. %s" % other)
-
-    def clear_content(self):
-        self.nodes = []
-
-
-class TemplateParser(object):
-    """Parse all blocks
-
-    Args:
-        text: text to parse
-        context: context to parse in
-        path: folder path to templates
-        writer: string of writer class to use
-        lexers: dict of custom lexers to use.
-        delimiters: for example `('{{','}}')`
-        _super_nodes: a list of nodes to check for inclusion
-            this should only be set by "self.extend"
-            It contains a list of SuperNodes from a child
-            template that need to be handled.
-
-    """
-
-    default_delimiters = ('{{', '}}')
-    r_tag = compile(r'(\{\{.*?\}\})', DOTALL)
-
-    r_multiline = compile(r'(""".*?""")|(\'\'\'.*?\'\'\')', DOTALL)
-
-    # These are used for re-indentation.
-    # Indent + 1
-    re_block = compile('^(elif |else:|except:|except |finally:).*$', DOTALL)
-
-    # Indent - 1
-    re_unblock = compile('^(return|continue|break|raise)( .*)?$', DOTALL)
-    # Indent - 1
-    re_pass = compile('^pass( .*)?$', DOTALL)
-
-    def __init__(self, text,
-                 name="ParserContainer",
-                 context=dict(),
-                 path='views/',
-                 writer='response.write',
-                 lexers={},
-                 delimiters=('{{', '}}'),
-                 _super_nodes = [],
-                 ):
-
-        # Keep a root level name.
-        self.name = name
-        # Raw text to start parsing.
-        self.text = text
-        # Writer to use (refer to the default for an example).
-        # This will end up as
-        # "%s(%s, escape=False)" % (self.writer, value)
-        self.writer = writer
-        # Dictionary of custom name lexers to use.
-        if isinstance(lexers, dict):
-            self.lexers = lexers
-        else:
-            self.lexers = {}
-
-        # Path of templates
-        self.path = path
-        # Context for templates.
-        self.context = context
-
-        # allow optional alternative delimiters
-        if delimiters != self.default_delimiters:
-            escaped_delimiters = (escape(delimiters[0]),
-                                  escape(delimiters[1]))
-            self.r_tag = compile(r'(%s.*?%s)' % escaped_delimiters, DOTALL)
-        elif hasattr(context.get('response', None), 'delimiters'):
-            if (context['response'].delimiters != self.default_delimiters) and (context['response'].delimiters != None):
-                delimiters = context['response'].delimiters
-                escaped_delimiters = (
-                    escape(delimiters[0]),
-                    escape(delimiters[1]))
-                self.r_tag = compile(r'(%s.*?%s)' % escaped_delimiters,
-                                     DOTALL)
-        self.delimiters = delimiters
-
-        # Create a root level Content that everything will go into.
-        self.content = Content(name=name)
-
-        # Stack will hold our current stack of nodes.
-        # As we descend into a node, it will be added to the stack
-        # And when we leave, it will be removed from the stack.
-        # self.content should stay on the stack at all times.
-        self.stack = [self.content]
-
-        # This variable will hold a reference to every super block
-        # that we come across in this template.
-        self.super_nodes = []
-
-        # This variable will hold a reference to the child
-        # super nodes that need handling.
-        self.child_super_nodes = _super_nodes
-
-        # This variable will hold a reference to every block
-        # that we come across in this template
-        self.blocks = {}
-
-        # Begin parsing.
-        self.parse(text)
-
-    def to_string(self):
-        """
-        Returns the parsed template with correct indentation.
-
-        Used to make it easier to port to python3.
-        """
-        return self.reindent(str(self.content))
-
-    def __str__(self):
-        "Makes sure str works exactly the same as python 3"
-        return self.to_string()
-
-    def __unicode__(self):
-        "Makes sure str works exactly the same as python 3"
-        return self.to_string()
-
-    def reindent(self, text):
-        """
-        Reindents a string of unindented python code.
-        """
-
-        # Get each of our lines into an array.
-        lines = text.split('\n')
-
-        # Our new lines
-        new_lines = []
-
-        # Keeps track of how many indents we have.
-        # Used for when we need to drop a level of indentation
-        # only to reindent on the next line.
-        credit = 0
-
-        # Current indentation
-        k = 0
-
-        #################
-        # THINGS TO KNOW
-        #################
-
-        # k += 1 means indent
-        # k -= 1 means unindent
-        # credit = 1 means unindent on the next line.
-
-        for raw_line in lines:
-            line = raw_line.strip()
-
-            # ignore empty lines
-            if not line:
-                continue
-
-            # If we have a line that contains python code that
-            # should be unindented for this line of code.
-            # and then reindented for the next line.
-            if TemplateParser.re_block.match(line):
-                k = k + credit - 1
-
-            # We obviously can't have a negative indentation
-            k = max(k, 0)
-
-            # Add the indentation!
-            new_lines.append(' ' * (4 * k) + line)
-
-            # Bank account back to 0 again :(
-            credit = 0
-
-            # If we are a pass block, we obviously de-dent.
-            if TemplateParser.re_pass.match(line):
-                k -= 1
-
-            # If we are any of the following, de-dent.
-            # However, we should stay on the same level
-            # But the line right after us will be de-dented.
-            # So we add one credit to keep us at the level
-            # while moving back one indentation level.
-            if TemplateParser.re_unblock.match(line):
-                credit = 1
-                k -= 1
-
-            # If we are an if statement, a try, or a semi-colon we
-            # probably need to indent the next line.
-            if line.endswith(':') and not line.startswith('#'):
-                k += 1
-
-        # This must come before so that we can raise an error with the
-        # right content.
-        new_text = '\n'.join(new_lines)
-
-        if k > 0:
-            self._raise_error('missing "pass" in view', new_text)
-        elif k < 0:
-            self._raise_error('too many "pass" in view', new_text)
-
-        return new_text
-
-    def _raise_error(self, message='', text=None):
-        """
-        Raises an error using itself as the filename and textual content.
-        """
-        raise RestrictedError(self.name, text or self.text, message)
-
-    def _get_file_text(self, filename):
-        """
-        Attempts to open ``filename`` and retrieve its text.
-
-        This will use self.path to search for the file.
-        """
-
-        # If they didn't specify a filename, how can we find one!
-        if not filename.strip():
-            self._raise_error('Invalid template filename')
-
-        # Allow Views to include other views dynamically
-        context = self.context
-        if current and "response" not in context:
-            context["response"] = getattr(current, 'response', None)
-
-        # Get the filename; filename looks like ``"template.html"``.
-        # We need to eval to remove the quotes and get the string type.
-        filename = eval(filename, context)
-
-        # Allow empty filename for conditional extend and include directives.
-        if not filename:
-            return ''
-
-        # Get the path of the file on the system.
-        filepath = self.path and os.path.join(self.path, filename) or filename
-
-        # try to read the text.
-        try:
-            fileobj = open(filepath, 'rb')
-            text = fileobj.read()
-            fileobj.close()
-        except IOError:
-            self._raise_error('Unable to open included view file: ' + filepath)
-        text = to_native(text)
-        return text
-
-    def include(self, content, filename):
-        """
-        Includes ``filename`` here.
-        """
-        text = self._get_file_text(filename)
-
-        t = TemplateParser(text,
-                           name=filename,
-                           context=self.context,
-                           path=self.path,
-                           writer=self.writer,
-                           delimiters=self.delimiters)
-
-        content.append(t.content)
-
-    def extend(self, filename):
-        """
-        Extends `filename`. Anything not declared in a block defined by the
-        parent will be placed in the parent templates `{{include}}` block.
-        """
-        # If no filename, create a dummy layout with only an {{include}}.
-        text = self._get_file_text(filename) or '%sinclude%s' % tuple(self.delimiters)
-
-        # Create out nodes list to send to the parent
-        super_nodes = []
-        # We want to include any non-handled nodes.
-        super_nodes.extend(self.child_super_nodes)
-        # And our nodes as well.
-        super_nodes.extend(self.super_nodes)
-
-        t = TemplateParser(text,
-                           name=filename,
-                           context=self.context,
-                           path=self.path,
-                           writer=self.writer,
-                           delimiters=self.delimiters,
-                           _super_nodes=super_nodes)
-
-        # Make a temporary buffer that is unique for parent
-        # template.
-        buf = BlockNode(
-            name='__include__' + filename, delimiters=self.delimiters)
-        pre = []
-
-        # Iterate through each of our nodes
-        for node in self.content.nodes:
-            # If a node is a block
-            if isinstance(node, BlockNode):
-                # That happens to be in the parent template
-                if node.name in t.content.blocks:
-                    # Do not include it
-                    continue
-
-            if isinstance(node, Node):
-                # Or if the node was before the extension
-                # we should not include it
-                if node.pre_extend:
-                    pre.append(node)
-                    continue
-
-            # Otherwise, it should go int the
-            # Parent templates {{include}} section.
-                buf.append(node)
-            else:
-                buf.append(node)
-
-        # Clear our current nodes. We will be replacing this with
-        # the parent nodes.
-        self.content.nodes = []
-
-        t_content = t.content
-
-        # Set our include, unique by filename
-        t_content.blocks['__include__' + filename] = buf
-
-        # Make sure our pre_extended nodes go first
-        t_content.insert(pre)
-
-        # Then we extend our blocks
-        t_content.extend(self.content)
-
-        # Work off the parent node.
-        self.content = t_content
-
-    def parse(self, text):
-
-        # Basically, r_tag.split will split the text into
-        # an array containing, 'non-tag', 'tag', 'non-tag', 'tag'
-        # so if we alternate this variable, we know
-        # what to look for. This is alternate to
-        # line.startswith("{{")
-        in_tag = False
-        extend = None
-        pre_extend = True
-
-        # Use a list to store everything in
-        # This is because later the code will "look ahead"
-        # for missing strings or brackets.
-        ij = self.r_tag.split(text)
-        # j = current index
-        # i = current item
-        stack = self.stack
-        for j in range(len(ij)):
-            i = ij[j]
-
-            if i:
-                if not stack:
-                    self._raise_error('The "end" tag is unmatched, please check if you have a starting "block" tag')
-
-                # Our current element in the stack.
-                top = stack[-1]
-
-                if in_tag:
-                    line = i
-
-                    # Get rid of delimiters
-                    line = line[len(self.delimiters[0]): \
-                                    -len(self.delimiters[1])].strip()
-
-                    # This is bad juju, but let's do it anyway
-                    if not line:
-                        continue
-
-                    # We do not want to replace the newlines in code,
-                    # only in block comments.
-                    def remove_newline(re_val):
-                        # Take the entire match and replace newlines with
-                        # escaped newlines.
-                        return re_val.group(0).replace('\n', '\\n')
-
-                    # Perform block comment escaping.
-                    # This performs escaping ON anything
-                    # in between """ and """
-                    line = sub(TemplateParser.r_multiline,
-                               remove_newline,
-                               line)
-
-                    if line.startswith('='):
-                        # IE: {{=response.title}}
-                        name, value = '=', line[1:].strip()
-                    else:
-                        v = line.split(' ', 1)
-                        if len(v) == 1:
-                            # Example
-                            # {{ include }}
-                            # {{ end }}
-                            name = v[0]
-                            value = ''
-                        else:
-                            # Example
-                            # {{ block pie }}
-                            # {{ include "layout.html" }}
-                            # {{ for i in range(10): }}
-                            name = v[0]
-                            value = v[1]
-
-                    # This will replace newlines in block comments
-                    # with the newline character. This is so that they
-                    # retain their formatting, but squish down to one
-                    # line in the rendered template.
-
-                    # First check if we have any custom lexers
-                    if name in self.lexers:
-                        # Pass the information to the lexer
-                        # and allow it to inject in the environment
-
-                        # You can define custom names such as
-                        # '{{<<variable}}' which could potentially
-                        # write unescaped version of the variable.
-                        self.lexers[name](parser=self,
-                                          value=value,
-                                          top=top,
-                                          stack=stack)
-
-                    elif name == '=':
-                        # So we have a variable to insert into
-                        # the template
-                        buf = "\n%s(%s)" % (self.writer, value)
-                        top.append(Node(buf, pre_extend=pre_extend))
-
-                    elif name == 'block' and not value.startswith('='):
-                        # Make a new node with name.
-                        node = BlockNode(name=value.strip(),
-                                         pre_extend=pre_extend,
-                                         delimiters=self.delimiters)
-
-                        # Append this node to our active node
-                        top.append(node)
-
-                        # Make sure to add the node to the stack.
-                        # so anything after this gets added
-                        # to this node. This allows us to
-                        # "nest" nodes.
-                        stack.append(node)
-
-                    elif name == 'end' and not value.startswith('='):
-                        # We are done with this node.
-
-                        # Save an instance of it
-                        self.blocks[top.name] = top
-
-                        # Pop it.
-                        stack.pop()
-
-                    elif name == 'super' and not value.startswith('='):
-                        # Get our correct target name
-                        # If they just called {{super}} without a name
-                        # attempt to assume the top blocks name.
-                        if value:
-                            target_node = value
-                        else:
-                            target_node = top.name
-
-                        # Create a SuperNode instance
-                        node = SuperNode(name=target_node,
-                                         pre_extend=pre_extend)
-
-                        # Add this to our list to be taken care of
-                        self.super_nodes.append(node)
-
-                        # And put in in the tree
-                        top.append(node)
-
-                    elif name == 'include' and not value.startswith('='):
-                        # If we know the target file to include
-                        if value:
-                            self.include(top, value)
-
-                        # Otherwise, make a temporary include node
-                        # That the child node will know to hook into.
-                        else:
-                            include_node = BlockNode(
-                                name='__include__' + self.name,
-                                pre_extend=pre_extend,
-                                delimiters=self.delimiters)
-                            top.append(include_node)
-
-                    elif name == 'extend' and not value.startswith('='):
-                        # We need to extend the following
-                        # template.
-                        extend = value
-                        pre_extend = False
-
-                    else:
-                        # If we don't know where it belongs
-                        # we just add it anyways without formatting.
-                        if line and in_tag:
-
-                            # Split on the newlines >.<
-                            tokens = line.split('\n')
-
-                            # We need to look for any instances of
-                            # for i in range(10):
-                            #   = i
-                            # pass
-                            # So we can properly put a response.write() in place.
-                            continuation = False
-                            len_parsed = 0
-                            for k, token in enumerate(tokens):
-
-                                token = tokens[k] = token.strip()
-                                len_parsed += len(token)
-
-                                if token.startswith('='):
-                                    if token.endswith('\\'):
-                                        continuation = True
-                                        tokens[k] = "\n%s(%s" % (
-                                            self.writer, token[1:].strip())
-                                    else:
-                                        tokens[k] = "\n%s(%s)" % (
-                                            self.writer, token[1:].strip())
-                                elif continuation:
-                                    tokens[k] += ')'
-                                    continuation = False
-
-                            buf = "\n%s" % '\n'.join(tokens)
-                            top.append(Node(buf, pre_extend=pre_extend))
-
-                else:
-                    # It is HTML so just include it.
-                    buf = "\n%s(%r, escape=False)" % (self.writer, i)
-                    top.append(Node(buf, pre_extend=pre_extend))
-
-            # Remember: tag, not tag, tag, not tag
-            in_tag = not in_tag
-
-        # Make a list of items to remove from child
-        to_rm = []
-
-        # Go through each of the children nodes
-        for node in self.child_super_nodes:
-            # If we declared a block that this node wants to include
-            if node.name in self.blocks:
-                # Go ahead and include it!
-                node.value = self.blocks[node.name]
-                # Since we processed this child, we don't need to
-                # pass it along to the parent
-                to_rm.append(node)
-
-        # Remove some of the processed nodes
-        for node in to_rm:
-            # Since this is a pointer, it works beautifully.
-            # Sometimes I miss C-Style pointers... I want my asterisk...
-            self.child_super_nodes.remove(node)
-
-        # If we need to extend a template.
-        if extend:
-            self.extend(extend)
-
-# We need this for integration with gluon
-
-
-def parse_template(filename,
-                   path='views/',
-                   context=dict(),
-                   lexers={},
-                   delimiters=('{{', '}}')
-                   ):
-    """
-    Args:
-        filename: can be a view filename in the views folder or an input stream
-        path: is the path of a views folder
-        context: is a dictionary of symbols used to render the template
-        lexers: dict of custom lexers to use
-        delimiters: opening and closing tags
-    """
-
-    # First, if we have a str try to open the file
-    if isinstance(filename, str):
-        fname = os.path.join(path, filename)
-        try:
-            with open(fname, 'rb') as fp:
-                text = fp.read()
-        except IOError:
-            raise RestrictedError(filename, '', 'Unable to find the file')
-    else:
-        text = filename.read()
-    text = to_native(text)
-    # Use the file contents to get a parsed template and return it.
-    return str(TemplateParser(text, context=context, path=path, lexers=lexers, delimiters=delimiters))
-
-
-def get_parsed(text):
-    """
-    Returns the indented python code of text. Useful for unit testing.
-
-    """
-    return str(TemplateParser(text))
-
-
-class DummyResponse():
-    def __init__(self):
-        self.body = StringIO()
-
-    def write(self, data, escape=True):
-        if not escape:
-            self.body.write(str(data))
-        elif hasattr(data, 'xml') and callable(data.xml):
-            self.body.write(data.xml())
-        else:
-            # make it a string
-            if not isinstance(data, (str, unicodeT)):
-                data = str(data)
-            elif isinstance(data, unicodeT):
-                data = data.encode('utf8', 'xmlcharrefreplace')
-            data = cgi.escape(data, True).replace("'", "&#x27;")
-            self.body.write(data)
-
-
-class NOESCAPE():
-    """
-    A little helper to avoid escaping.
-    """
-    def __init__(self, text):
-        self.text = text
-
-    def xml(self):
-        return self.text
-
-# And this is a generic render function.
-# Here for integration with gluon.
-
-
-def render(content="hello world",
-           stream=None,
-           filename=None,
-           path=None,
-           context={},
-           lexers={},
-           delimiters=('{{', '}}'),
-           writer='response.write'
-           ):
-    """
-    Generic render function
-
-    Args:
-        content: default content
-        stream: file-like obj to read template from
-        filename: where to find template
-        path: base path for templates
-        context: env
-        lexers: custom lexers to use
-        delimiters: opening and closing tags
-        writer: where to inject the resulting stream
-
-    Example::
-        >>> render()
-        'hello world'
-        >>> render(content='abc')
-        'abc'
-        >>> render(content="abc'")
-        "abc'"
-        >>> render(content=''''a"'bc''')
-        'a"'bc'
-        >>> render(content='a\\nbc')
-        'a\\nbc'
-        >>> render(content='a"bcd"e')
-        'a"bcd"e'
-        >>> render(content="'''a\\nc'''")
-        "'''a\\nc'''"
-        >>> render(content="'''a\\'c'''")
-        "'''a\'c'''"
-        >>> render(content='{{for i in range(a):}}{{=i}}<br />{{pass}}', context=dict(a=5))
-        '0<br />1<br />2<br />3<br />4<br />'
-        >>> render(content='{%for i in range(a):%}{%=i%}<br />{%pass%}', context=dict(a=5),delimiters=('{%','%}'))
-        '0<br />1<br />2<br />3<br />4<br />'
-        >>> render(content="{{='''hello\\nworld'''}}")
-        'hello\\nworld'
-        >>> render(content='{{for i in range(3):\\n=i\\npass}}')
-        '012'
-
-    """
-    # here to avoid circular Imports
-    try:
-        from gluon.globals import Response
-    except ImportError:
-        # Working standalone. Build a mock Response object.
-        Response = DummyResponse
-
-        # Add it to the context so we can use it.
-        if 'NOESCAPE' not in context:
-            context['NOESCAPE'] = NOESCAPE
-
-    if isinstance(content, unicodeT):
-        content = content.encode('utf8')
-
-    # save current response class
-    if context and 'response' in context:
-        old_response_body = context['response'].body
-        context['response'].body = StringIO()
-    else:
-        old_response_body = None
-        context['response'] = Response()
-
-    # If we don't have anything to render, why bother?
-    if not content and not stream and not filename:
-        raise SyntaxError("Must specify a stream or filename or content")
-
-    # Here for legacy purposes, probably can be reduced to
-    # something more simple.
-    close_stream = False
-    if not stream:
-        if filename:
-            stream = open(filename, 'rb')
-            close_stream = True
-        elif content:
-            stream = StringIO(to_native(content))
-
-    # Execute the template.
-    code = str(TemplateParser(stream.read(
-    ), context=context, path=path, lexers=lexers, delimiters=delimiters, writer=writer))
-
-    try:
-        exec(code, context)
-    except Exception:
-        # for i,line in enumerate(code.split('\n')): print i,line
-        raise
-
-    if close_stream:
-        stream.close()
-
-    # Returned the rendered content.
-    text = context['response'].body.getvalue()
-    if old_response_body is not None:
-        context['response'].body = old_response_body
-    return text

gluon/template.py

@@ -0,0 +1 @@
+packages/template/template.py

gluon/tests/test_cron.py

@@ -11,7 +11,7 @@ class TestCron(unittest.TestCase):
     def test_Token(self):
         appname_path = os.path.join(os.getcwd(), 'applications', 'welcome')
         t = Token(path=appname_path)
-        self.assertNotEqual(t.acquire(), None)
+        self.assertNotEqual(t.acquire(startup=True), None)
         self.assertFalse(t.release())
         self.assertEqual(t.acquire(), None)
         self.assertTrue(t.release())

gluon/tests/test_globals.py

@@ -231,6 +231,25 @@ class testResponse(unittest.TestCase):
         cookie = str(current.response.cookies)
         self.assertTrue('httponly' not in cookie.lower())
 
+    def test_cookies_samesite(self):
+        # Test Lax is the default mode
+        current = setup_clean_session()
+        current.session._fixup_before_save()
+        cookie = str(current.response.cookies)
+        self.assertTrue('samesite=lax' in cookie.lower())
+        # Test you can disable samesite
+        current = setup_clean_session()
+        current.session.samesite(False)
+        current.session._fixup_before_save()
+        cookie = str(current.response.cookies)
+        self.assertTrue('samesite' not in cookie.lower())
+        # Test you can change mode
+        current = setup_clean_session()
+        current.session.samesite('Strict')
+        current.session._fixup_before_save()
+        cookie = str(current.response.cookies)
+        self.assertTrue('samesite=strict' in cookie.lower())
+
     def test_include_meta(self):
         response = Response()
         response.meta[u'web2py'] = 'web2py'

gluon/tests/test_sqlhtml.py

@@ -4,6 +4,7 @@
 """
     Unit tests for gluon.sqlhtml
 """
+import datetime
 import os
 import sys
 import unittest
@@ -312,6 +313,25 @@ class TestSQLFORM(unittest.TestCase):
                                        Field('field_two', 'string'))
         self.assertEqual(factory_form.xml()[:5], b'<form')
 
+    def test_factory_applies_default_validators(self):
+        from gluon import current
+
+        factory_form = SQLFORM.factory(
+            Field('a_date', type='date'),
+        )
+        # Fake user input
+        current.request.post_vars.update({
+            '_formname': 'no_table/create',
+            'a_date': '2018-09-14',
+            '_formkey': '123',
+
+        })
+        # Fake the formkey
+        current.session['_formkey[no_table/create]'] = ['123']
+
+        self.assertTrue(factory_form.process().accepted)
+        self.assertIsInstance(factory_form.vars.a_date, datetime.date)
+
     #  def test_build_query(self):
     #     pass
 

gluon/tests/test_template.py

@@ -24,11 +24,11 @@ class TestTemplate(unittest.TestCase):
         self.assertEqual(render(content='"abc"'), '"abc"')
         self.assertEqual(render(content='"a\'bc"'), '"a\'bc"')
         self.assertEqual(render(content='"a\"bc"'), '"a\"bc"')
-        self.assertEqual(render(content=r'''"a\"bc"'''), r'"a\"bc"')
+        self.assertEqual(render(content=r'"a\"bc"'), r'"a\"bc"')
-        self.assertEqual(render(content=r'''"""abc\""""'''), r'"""abc\""""')
+        self.assertEqual(render(content=r'"""abc\""""'), r'"""abc\""""')
 
     def testEqualWrite(self):
-        "test generation of response.write from ="
+        "test generation of response.write"
         self.assertEqual(render(content='{{=2+2}}'), '4')
         self.assertEqual(render(content='{{="abc"}}'), 'abc')
         # whitespace is stripped
@@ -81,55 +81,55 @@ class TestTemplate(unittest.TestCase):
                 else:
                     setattr(module, fn_name, unpatch)
 
-        def dummy_open(path, mode):
+        def dummy_open(path):
             if path == pjoin('views', 'layout.html'):
-                return StringIO("{{block left_sidebar}}left{{end}}"
+                return ("{{block left_sidebar}}left{{end}}"
-                                "{{include}}"
+                        "{{include}}" 
-                                "{{block right_sidebar}}right{{end}}")
+                        "{{block right_sidebar}}right{{end}}")
             elif path == pjoin('views', 'layoutbrackets.html'):
-                return StringIO("[[block left_sidebar]]left[[end]]"
+                return ("[[block left_sidebar]]left[[end]]"
-                                "[[include]]"
+                        "[[include]]"
-                                "[[block right_sidebar]]right[[end]]")
+                        "[[block right_sidebar]]right[[end]]")
             elif path == pjoin('views', 'default', 'index.html'):
-                return StringIO("{{extend 'layout.html'}}"
+                return ("{{extend 'layout.html'}}"
-                                "{{block left_sidebar}}{{super}} {{end}}"
+                        "{{block left_sidebar}}{{super}} {{end}}"
-                                "to"
+                        "to"
-                                "{{block right_sidebar}} {{super}}{{end}}")
+                        "{{block right_sidebar}} {{super}}{{end}}")
             elif path == pjoin('views', 'default', 'indexbrackets.html'):
-                return StringIO("[[extend 'layoutbrackets.html']]"
+                return ("[[extend 'layoutbrackets.html']]"
-                                "[[block left_sidebar]][[super]] [[end]]"
+                        "[[block left_sidebar]][[super]] [[end]]"
-                                "to"
+                        "to"
-                                "[[block right_sidebar]] [[super]][[end]]")
+                        "[[block right_sidebar]] [[super]][[end]]")
             elif path == pjoin('views', 'default', 'missing.html'):
-                return StringIO("{{extend 'wut'}}"
+                return ("{{extend 'wut'}}"
-                                "{{block left_sidebar}}{{super}} {{end}}"
+                        "{{block left_sidebar}}{{super}} {{end}}"
-                                "to"
+                        "to"
-                                "{{block right_sidebar}} {{super}}{{end}}")
+                        "{{block right_sidebar}} {{super}}{{end}}")
             elif path == pjoin('views', 'default', 'noescape.html'):
-                return StringIO("""{{=NOESCAPE('<script></script>')}}""")
+                return "{{=NOESCAPE('<script></script>')}}"
             raise IOError
         
-        with monkey_patch(template, 'open', dummy_open):
+        self.assertEqual(
-            self.assertEqual(
+            render(filename=pjoin('views', 'default', 'index.html'),
-                render(filename=pjoin('views', 'default', 'index.html'),
+                   path='views', reader=dummy_open),
-                       path='views'),
+            'left to right')
-                'left to right')
+        self.assertEqual(
-            self.assertEqual(
+            render(filename=pjoin('views', 'default', 'indexbrackets.html'),
-                render(filename=pjoin('views', 'default', 'indexbrackets.html'),
+                   path='views', delimiters='[[ ]]', reader=dummy_open),
-                       path='views', delimiters=('[[', ']]')),
+            'left to right')
-                'left to right')
+        self.assertRaises(
-            self.assertRaises(
+            RestrictedError,
-                RestrictedError,
+            render,
-                render,
+            filename=pjoin('views', 'default', 'missing.html'),
-                filename=pjoin('views', 'default', 'missing.html'),
+            path='views',
-                path='views')
+            reader=dummy_open)
-            response = template.DummyResponse()
+        response = template.DummyResponse()
-            response.delimiters = ('[[', ']]')
+        response.delimiters = ('[[', ']]')
-            self.assertEqual(
+        self.assertEqual(
-                render(filename=pjoin('views', 'default', 'indexbrackets.html'),
+            render(filename=pjoin('views', 'default', 'indexbrackets.html'),
-                       path='views', context={'response': response}),
+                   path='views', context={'response': response}, reader=dummy_open),
-                'left to right')
+            'left to right')
-            self.assertEqual(
+        self.assertEqual(
-                render(filename=pjoin('views', 'default', 'noescape.html'),
+            render(filename=pjoin('views', 'default', 'noescape.html'),
-                       context={'NOESCAPE': template.NOESCAPE}),
+                   context={'NOESCAPE': template.NOESCAPE}, reader=dummy_open),
-                '<script></script>')
+            '<script></script>')

gluon/tests/test_tools.py

@@ -193,7 +193,7 @@ class TestMail(unittest.TestCase):
         message = TestMail.DummySMTP.inbox.pop()
         attachment = message.parsed_payload.get_payload(1).get_payload(decode=True)
         with open(module_file, 'rb') as mf:
-            self.assertEqual(attachment.decode('utf-8'), mf.read().decode('utf-8'))
+            self.assertEqual(attachment, mf.read())
         # Test missing attachment name error
         stream = open(module_file)
         self.assertRaises(Exception, lambda *args, **kwargs: Mail.Attachment(*args, **kwargs), stream)

gluon/tests/test_validators.py

@@ -977,6 +977,8 @@ class TestValidators(unittest.TestCase):
         self.assertEqual(rtn, (img, 'oops'))
         rtn = IS_IMAGE(error_message='oops', minsize=(100, 50))(img)
         self.assertEqual(rtn, (img, 'oops'))
+        rtn = IS_IMAGE(error_message='oops', aspectratio=(1, 1))(img)
+        self.assertEqual(rtn, (img, 'oops'))
 
         img = DummyImageFile('test', 'gif', 50, 100)
         rtn = IS_IMAGE()(img)
@@ -985,6 +987,8 @@ class TestValidators(unittest.TestCase):
         self.assertEqual(rtn, (img, 'oops'))
         rtn = IS_IMAGE(error_message='oops', minsize=(100, 50))(img)
         self.assertEqual(rtn, (img, 'oops'))
+        rtn = IS_IMAGE(error_message='oops', aspectratio=(1, 1))(img)
+        self.assertEqual(rtn, (img, 'oops'))
 
         img = DummyImageFile('test', 'jpeg', 50, 100)
         rtn = IS_IMAGE()(img)
@@ -993,6 +997,8 @@ class TestValidators(unittest.TestCase):
         self.assertEqual(rtn, (img, 'oops'))
         rtn = IS_IMAGE(error_message='oops', minsize=(100, 50))(img)
         self.assertEqual(rtn, (img, 'oops'))
+        rtn = IS_IMAGE(error_message='oops', aspectratio=(1, 1))(img)
+        self.assertEqual(rtn, (img, 'oops'))
 
         img = DummyImageFile('test', 'png', 50, 100)
         rtn = IS_IMAGE()(img)
@@ -1001,6 +1007,8 @@ class TestValidators(unittest.TestCase):
         self.assertEqual(rtn, (img, 'oops'))
         rtn = IS_IMAGE(error_message='oops', minsize=(100, 50))(img)
         self.assertEqual(rtn, (img, 'oops'))
+        rtn = IS_IMAGE(error_message='oops', aspectratio=(1, 1))(img)
+        self.assertEqual(rtn, (img, 'oops'))
 
         img = DummyImageFile('test', 'xls', 50, 100)
         rtn = IS_IMAGE(error_message='oops')(img)

gluon/tools.py

@@ -905,14 +905,14 @@ class Recaptcha2(DIV):
         })
         request = urllib2.Request(
             url=self.VERIFY_SERVER,
-            data=params,
+            data=to_bytes(params),
             headers={'Content-type': 'application/x-www-form-urlencoded',
                      'User-agent': 'reCAPTCHA Python'})
         httpresp = urllib2.urlopen(request)
         content = httpresp.read()
         httpresp.close()
         try:
-            response_dict = json.loads(content)
+            response_dict = json.loads(to_native(content))
         except:
             self.errors['captcha'] = self.error_message
             return False
@@ -1378,6 +1378,7 @@ class Auth(AuthAPI):
                             login_after_password_change=True,
                             login_after_registration=False,
                             login_captcha=None,
+                            login_specify_error=False,
                             long_expiration=3600 * 30 * 24,  # one month
                             mailer=None,
                             manager_actions={},
@@ -2251,7 +2252,7 @@ class Auth(AuthAPI):
             if basic_auth_realm:
                 raise http_401
             return (True, False, False)
-        (username, sep, password) = base64.b64decode(basic[6:]).partition(':')
+        (username, sep, password) = base64.b64decode(basic[6:]).partition(b':')
         is_valid_user = sep and self.login_bare(username, password)
         if not is_valid_user and basic_auth_realm:
             raise http_401
@@ -2567,6 +2568,8 @@ class Auth(AuthAPI):
                            settings.formstyle, 'captcha__row')
                 accepted_form = False
 
+                specific_error = self.messages.invalid_user
+
                 if form.accepts(request, session if self.csrf_prevention else None,
                                 formname='login', dbio=False,
                                 onvalidation=onvalidation,
@@ -2582,6 +2585,7 @@ class Auth(AuthAPI):
                         user = table_user(**{username: entered_username})
                     if user:
                         # user in db, check if registration pending or disabled
+                        specific_error = self.messages.invalid_password
                         temp_user = user
                         if (temp_user.registration_key or '').startswith('pending'):
                             response.flash = self.messages.registration_pending
@@ -2631,7 +2635,7 @@ class Auth(AuthAPI):
                         self.log_event(self.messages['login_failed_log'],
                                        request.post_vars)
                         # invalid login
-                        session.flash = self.messages.invalid_login
+                        session.flash = specific_error if self.settings.login_specify_error else self.messages.invalid_login
                         callback(onfail, None)
                         redirect(
                             self.url(args=request.args, vars=request.get_vars),
@@ -3447,7 +3451,8 @@ class Auth(AuthAPI):
         if log is DEFAULT:
             log = self.messages['reset_password_log']
         userfield = self.settings.login_userfield or 'username' \
-            if 'username' in table_user.fields else 'email'
+            if self.settings.login_userfield or 'username' \
+            in table_user.fields else 'email'
         if userfield == 'email':
             table_user.email.requires = [
                 IS_EMAIL(error_message=self.messages.invalid_email),
@@ -3455,7 +3460,7 @@ class Auth(AuthAPI):
                          error_message=self.messages.invalid_email)]
             if not self.settings.email_case_sensitive:
                 table_user.email.requires.insert(0, IS_LOWER())
-        else:
+        elif userfield == 'username':
             table_user.username.requires = [
                 IS_IN_DB(self.db, table_user.username,
                          error_message=self.messages.invalid_username)]

gluon/utils.py

@@ -462,3 +462,51 @@ def local_html_escape(data, quote=False):
             data = data.replace(b'"', b""")
             data = data.replace(b'\'', b"'")
         return data
+
+
+def unlocalised_http_header_date(data):
+    """
+    Converts input datetime to format defined by RFC 7231, section 7.1.1.1
+
+    Previously, %a and %b formats were used for weekday and month names, but
+    those are not locale-safe. uWSGI requires latin1-encodable headers and
+    for example in cs_CS locale, fourth day in week is not encodable in latin1,
+    as it's "Čt".
+
+    Example output: Sun, 06 Nov 1994 08:49:37 GMT
+    """
+
+    short_weekday = {
+        "0": "Sun",
+        "1": "Mon",
+        "2": "Tue",
+        "3": "Wed",
+        "4": "Thu",
+        "5": "Fri",
+        "6": "Sat",
+    }.get(time.strftime("%w", data))
+
+    day_of_month = time.strftime("%d", data)
+
+    short_month = {
+        "01": "Jan",
+        "02": "Feb",
+        "03": "Mar",
+        "04": "Apr",
+        "05": "May",
+        "06": "Jun",
+        "07": "Jul",
+        "08": "Aug",
+        "09": "Sep",
+        "10": "Oct",
+        "11": "Nov",
+        "12": "Dec",
+    }.get(time.strftime("%m", data))
+
+    year_and_time = time.strftime("%Y %H:%M:%S GMT")
+
+    return "{}, {} {} {}".format(
+        short_weekday,
+        day_of_month,
+        short_month,
+        year_and_time)

gluon/widget.py

@@ -1288,17 +1288,18 @@ end tell
         line = py2exe_getline(filename, lineno, *args, **kwargs)
         if not line:
             try:
-                f = open(filename, "r")
+                f = open(filename, "rb")
                 try:
                     for i, line in enumerate(f):
+                        line = line.decode('utf-8')
                         if lineno == i + 1:
                             break
                     else:
-                        line = None
+                        line = ''
                 finally:
                     f.close()
             except (IOError, OSError):
-                line = None
+                line = ''
         return line
     linecache.getline = getline
 

gluon/xmlrpc.py

@@ -7,8 +7,12 @@
 | License: LGPLv3 (http://www.gnu.org/licenses/lgpl.html)
 """
 
-from SimpleXMLRPCServer import SimpleXMLRPCDispatcher
+from gluon._compat import PY2
 
+if PY2:
+    from SimpleXMLRPCServer import SimpleXMLRPCDispatcher
+else:
+    from xmlrpc.server import SimpleXMLRPCDispatcher
 
 def handler(request, response, methods):
     response.session_id = None  # no sessions for xmlrpc

scripts/make_min_web2py.py

@@ -48,7 +48,7 @@ def main():
     global REQUIRED, IGNORED
 
     if len(sys.argv) < 2:
-        print USAGE
+        print(USAGE)
 
     # make target folder
     target = sys.argv[1]