From 22c89d8dccad2f8e28e7246f214f8cba1ea0802b Mon Sep 17 00:00:00 2001
From: mdipierro <massimo.dipierro@gmail.com>
Date: Thu, 17 Dec 2015 21:19:08 -0600
Subject: [PATCH] version 2.13.1

---
 CHANGELOG                   | 18 ++++++++---
 Makefile                    |  2 +-
 gluon/admin.py              |  2 +-
 gluon/cache.py              |  4 +--
 gluon/compileapp.py         | 60 ++++++++++++++++++-------------------
 gluon/custom_import.py      |  4 +--
 gluon/template.py           |  2 +-
 gluon/tests/test_globals.py | 26 ++++++++--------
 gluon/tests/test_html.py    |  2 +-
 gluon/tests/test_storage.py |  2 +-
 gluon/tests/test_utils.py   | 22 +++++++-------
 gluon/tools.py              | 40 ++++++++++++-------------
 gluon/validators.py         |  2 +-
 13 files changed, 98 insertions(+), 88 deletions(-)

diff --git a/CHANGELOG b/CHANGELOG
index 5972aeb2..07b74553 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,8 +1,18 @@
-## master
-- Change .flash and .hidden to .w2p_flash and .w2p_hidden
-     - copy the static folder from the new welcome app when you update web2py
+## 2.13.1
 
-## 2.12.1
+- added fabfile.py
+- fixed oauth2 renew token, thanks dokime7
+- fixed add_membership, del_membership, add_membership IntegrityError (when auth.enable_record_versioning) 
+- allow passing unicode to template render
+- allow IS_NOT_IN_DB to work with custom primarykey, thanks timmyborg
+- allow HttpOnly cookies
+- french pluralizaiton rules, thanks Mathieu Clabaut
+- fixed bug in redirect to cas service, thanks Fernando González
+- allow deploying to pythonanywhere from the web2py admin that you're running locally, thanks Leonel
+- better tests
+- many more bug fixes
+
+## 2.12.1-3
 
 - security fix: Validate for open redirect everywhere, not just in login()
 - allow to pack invidual apps and selected files as packed exe files
diff --git a/Makefile b/Makefile
index 7e3a9db1..1149f2f3 100644
--- a/Makefile
+++ b/Makefile
@@ -32,7 +32,7 @@ update:
 	echo "remember that pymysql was tweaked"
 src:
 	### Use semantic versioning
-	echo 'Version 2.12.3-stable+timestamp.'`date +%Y.%m.%d.%H.%M.%S` > VERSION
+	echo 'Version 2.13.1-stable+timestamp.'`date +%Y.%m.%d.%H.%M.%S` > VERSION
 	### rm -f all junk files
 	make clean
 	### clean up baisc apps
diff --git a/gluon/admin.py b/gluon/admin.py
index d1b1f429..cf8627a3 100644
--- a/gluon/admin.py
+++ b/gluon/admin.py
@@ -120,7 +120,7 @@ def app_cleanup(app, request):
                 r = False
 
     # Remove cache files
-    path = apath('%s/cache/' % app, request)    
+    path = apath('%s/cache/' % app, request)
     if os.path.exists(path):
         CacheOnDisk(folder=path).clear()
         for f in os.listdir(path):
diff --git a/gluon/cache.py b/gluon/cache.py
index c2f9a488..814331cc 100644
--- a/gluon/cache.py
+++ b/gluon/cache.py
@@ -58,7 +58,7 @@ def remove_oldest_entries(storage, percentage=90):
     # compute current memory usage (%)
     old_mem = psutil.virtual_memory().percent
     # if we have data in storage and utilization exceeds 90%
-    while storage and old_mem > percentage:    
+    while storage and old_mem > percentage:
         # removed oldest entry
         storage.popitem(last=False)
         # garbage collect
@@ -378,7 +378,7 @@ class CacheOnDisk(CacheAbstract):
 
 
         def safe_apply(self, key, function, default_value=None):
-            """ 
+            """
             Safely apply a function to the value of a key in storage and set
             the return value of the function to it.
 
diff --git a/gluon/compileapp.py b/gluon/compileapp.py
index c6bbba3b..c7e07200 100644
--- a/gluon/compileapp.py
+++ b/gluon/compileapp.py
@@ -470,21 +470,21 @@ def compile_views(folder, skip_failed_views=False):
     """
 
     path = pjoin(folder, 'views')
-    failed_views = [] 
+    failed_views = []
     for fname in listdir(path, '^[\w/\-]+(\.\w+)*$'):
         try:
             data = parse_template(fname, path)
         except Exception, e:
-            if skip_failed_views:                                                                            
-                failed_views.append(file)                                                                    
-            else:                                                                                            
-                raise Exception("%s in %s" % (e, file))                                                      
-        else:                                                                                                
-            filename = ('views/%s.py' % file).replace('/', '_').replace('\\', '_')                           
-            filename = pjoin(folder, 'compiled', filename)                                                   
-            write_file(filename, data)                                                                       
-            save_pyc(filename)                                                                               
-            os.unlink(filename)                                                                              
+            if skip_failed_views:
+                failed_views.append(file)
+            else:
+                raise Exception("%s in %s" % (e, file))
+        else:
+            filename = ('views/%s.py' % file).replace('/', '_').replace('\\', '_')
+            filename = pjoin(folder, 'compiled', filename)
+            write_file(filename, data)
+            save_pyc(filename)
+            os.unlink(filename)
     return failed_views if failed_views else None
 
 
@@ -675,25 +675,25 @@ def run_view_in(environment):
         restricted(ccode, environment, 'file stream')
     else:
         filename = pjoin(folder, 'views', view)
-        if os.path.exists(path): # compiled views                                                            
-            x = view.replace('/', '_')                                                                       
-            files = ['views_%s.pyc' % x]                                                                     
-            is_compiled = os.path.exists(pjoin(path, files[0]))                                              
-            # Don't use a generic view if the non-compiled view exists.                                      
-            if is_compiled or (not is_compiled and not os.path.exists(filename)):                            
-                if allow_generic:                                                                            
-                    files.append('views_generic.%s.pyc' % request.extension)                                 
-                # for backward compatibility                                                                 
-                if request.extension == 'html':                                                              
-                    files.append('views_%s.pyc' % x[:-5])                                                    
-                    if allow_generic:                                                                        
-                        files.append('views_generic.pyc')                                                    
-                # end backward compatibility code                                                            
-                for f in files:                                                                              
-                    compiled = pjoin(path, f)                                                                
-                    if os.path.exists(compiled):                                                             
-                        code = read_pyc(compiled)                                                            
-                        restricted(code, environment, layer=compiled)                                        
+        if os.path.exists(path): # compiled views
+            x = view.replace('/', '_')
+            files = ['views_%s.pyc' % x]
+            is_compiled = os.path.exists(pjoin(path, files[0]))
+            # Don't use a generic view if the non-compiled view exists.
+            if is_compiled or (not is_compiled and not os.path.exists(filename)):
+                if allow_generic:
+                    files.append('views_generic.%s.pyc' % request.extension)
+                # for backward compatibility
+                if request.extension == 'html':
+                    files.append('views_%s.pyc' % x[:-5])
+                    if allow_generic:
+                        files.append('views_generic.pyc')
+                # end backward compatibility code
+                for f in files:
+                    compiled = pjoin(path, f)
+                    if os.path.exists(compiled):
+                        code = read_pyc(compiled)
+                        restricted(code, environment, layer=compiled)
                         return
         if not os.path.exists(filename) and allow_generic:
             view = 'generic.' + request.extension
diff --git a/gluon/custom_import.py b/gluon/custom_import.py
index f7bd91a2..cf677fda 100644
--- a/gluon/custom_import.py
+++ b/gluon/custom_import.py
@@ -41,7 +41,7 @@ class CustomImportException(ImportError):
 
 def custom_importer(name, globals=None, locals=None, fromlist=None, level=-1):
     """
-    web2py's custom importer. It behaves like the standard Python importer but 
+    web2py's custom importer. It behaves like the standard Python importer but
     it tries to transform import statements as something like
     "import applications.app_name.modules.x".
     If the import fails, it falls back on naive_importer
@@ -80,7 +80,7 @@ def custom_importer(name, globals=None, locals=None, fromlist=None, level=-1):
                 if not fromlist:
                     # import like "import x" or "import x.y"
                     result = None
-                    for itemname in name.split("."):                        
+                    for itemname in name.split("."):
                         new_mod = base_importer(
                             modules_prefix, globals, locals, [itemname], level)
                         try:
diff --git a/gluon/template.py b/gluon/template.py
index 3bbae113..0e297e92 100644
--- a/gluon/template.py
+++ b/gluon/template.py
@@ -900,7 +900,7 @@ def render(content="hello world",
 
     if isinstance(content, unicode):
         content = content.encode('utf8')
-            
+
     # save current response class
     if context and 'response' in context:
         old_response_body = context['response'].body
diff --git a/gluon/tests/test_globals.py b/gluon/tests/test_globals.py
index 90222bdc..b6040ac2 100644
--- a/gluon/tests/test_globals.py
+++ b/gluon/tests/test_globals.py
@@ -16,19 +16,19 @@ from gluon.globals import Request, Response, Session
 from gluon import URL
 
 def setup_clean_session():
-	request = Request(env={})
-	request.application = 'a'
-	request.controller = 'c'
-	request.function = 'f'
-	request.folder = 'applications/admin'
-	response = Response()
-	session = Session()
-	session.connect(request, response)
-	from gluon.globals import current
-	current.request = request
-	current.response = response
-	current.session = session
-	return current
+        request = Request(env={})
+        request.application = 'a'
+        request.controller = 'c'
+        request.function = 'f'
+        request.folder = 'applications/admin'
+        response = Response()
+        session = Session()
+        session.connect(request, response)
+        from gluon.globals import current
+        current.request = request
+        current.response = response
+        current.session = session
+        return current
 
 class testResponse(unittest.TestCase):
 
diff --git a/gluon/tests/test_html.py b/gluon/tests/test_html.py
index 27c46cbe..d426ea9c 100644
--- a/gluon/tests/test_html.py
+++ b/gluon/tests/test_html.py
@@ -309,7 +309,7 @@ class TestBareHelpers(unittest.TestCase):
         self.assertEqual(XML('<h1>Hello<a data-hello="world">World</a></h1>', sanitize=True),
             XML('<h1>HelloWorld</h1>'))
         #bug check for the sanitizer for closing no-close tags
-        self.assertEqual(XML('<p>Test</p><br/><p>Test</p><br/>', sanitize=True), 
+        self.assertEqual(XML('<p>Test</p><br/><p>Test</p><br/>', sanitize=True),
             XML('<p>Test</p><br /><p>Test</p><br />'))
 
     def testTAG(self):
diff --git a/gluon/tests/test_storage.py b/gluon/tests/test_storage.py
index ca047e95..86dcf231 100644
--- a/gluon/tests/test_storage.py
+++ b/gluon/tests/test_storage.py
@@ -146,7 +146,7 @@ class TestList(unittest.TestCase):
                          'something')
         # except if default is especified
         self.assertEqual(b(0, default=0, otherwise=lambda: 'something'), 0)
-        
+
     def test_listgetitem(self):
         '''Mantains list behaviour.'''
         a = List((1, 2, 3))
diff --git a/gluon/tests/test_utils.py b/gluon/tests/test_utils.py
index bccf6ce8..8f50cee1 100644
--- a/gluon/tests/test_utils.py
+++ b/gluon/tests/test_utils.py
@@ -24,42 +24,42 @@ class TestUtils(unittest.TestCase):
 
         data = md5_hash("web2py rocks")
         self.assertEqual(data, '79509f3246a2824dee64635303e99204')
-        
+
     def test_compare(self):
         """ Tests the compare funciton """
-        
+
         a, b = 'test123', 'test123'
         compare_result_true = compare(a, b)
         self.assertTrue(compare_result_true)
-        
+
         a, b = 'test123', 'test456'
         compare_result_false = compare(a, b)
         self.assertFalse(compare_result_false)
-    
+
     def test_simple_hash(self):
         """ Tests the simple_hash function """
-        
+
         # no key, no salt, md5
         data_md5 = simple_hash('web2py rocks!', key='', salt='', digest_alg='md5')
         self.assertEqual(data_md5, '37d95defba6c8834cb8cae86ee888568')
-        
+
         # no key, no salt, sha1
         data_sha1 = simple_hash('web2py rocks!', key='', salt='', digest_alg='sha1')
         self.assertEqual(data_sha1, '00489a46753d8db260c71542611cdef80652c4b7')
-        
+
         # no key, no salt, sha224
         data_sha224 = simple_hash('web2py rocks!', key='', salt='', digest_alg='sha224')
         self.assertEqual(data_sha224, '84d7054271842c2c17983baa2b1447e0289d101140a8c002d49d60da')
-        
+
         # no key, no salt, sha256
         data_sha256 = simple_hash('web2py rocks!', key='', salt='', digest_alg='sha256')
         self.assertEqual(data_sha256, '0849f224d8deb267e4598702aaec1bd749e6caec90832469891012a4be24af08')
-        
+
         # no key, no salt, sha384
         data_sha384 = simple_hash('web2py rocks!', key='', salt='', digest_alg='sha384')
-        self.assertEqual(data_sha384, 
+        self.assertEqual(data_sha384,
             '3cffaf39371adbe84eb10f588d2718207d8e965e9172a27a278321b86977351376ae79f92e91d8c58cad86c491282d5f')
-        
+
         # no key, no salt, sha512
         data_sha512 = simple_hash('web2py rocks!', key='', salt='', digest_alg='sha512')
         self.assertEqual(data_sha512, 'fa3237f594743e1d7b6c800bb134b3255cf4a98ab8b01e2ec23256328c9f8059'
diff --git a/gluon/tools.py b/gluon/tools.py
index e55b7da4..1f02c859 100644
--- a/gluon/tools.py
+++ b/gluon/tools.py
@@ -1426,7 +1426,7 @@ class Auth(object):
                  csrf_prevention=True, propagate_extension=None,
                  url_index=None):
 
-        ## next two lines for backward compatibility        
+        ## next two lines for backward compatibility
         if not db and environment and isinstance(environment, DAL):
             db = environment
         self.db = db
@@ -1544,7 +1544,7 @@ class Auth(object):
             self.define_signature()
         else:
             self.signature = None
-            
+
     def get_vars_next(self):
         next = current.request.vars._next
         if isinstance(next, (list, tuple)):
@@ -1554,7 +1554,7 @@ class Auth(object):
             # _next variable in the request.
             items = next.split('/')
             if '//' in next and items[2] != current.request.env.http_host:
-                next = None            
+                next = None
         return next
 
     def _get_user_id(self):
@@ -1611,7 +1611,7 @@ class Auth(object):
                        'retrieve_username', 'retrieve_password',
                        'reset_password', 'request_reset_password',
                        'change_password', 'profile', 'groups',
-                       'impersonate', 'not_authorized', 'confirm_registration', 
+                       'impersonate', 'not_authorized', 'confirm_registration',
                        'bulk_register','manage_tokens'):
             if len(request.args) >= 2 and args[0] == 'impersonate':
                 return getattr(self, args[0])(request.args[1])
@@ -2373,7 +2373,7 @@ class Auth(object):
         and a raw password.
         """
         settings = self._get_login_settings()
-        # users can register_bare even if no password is provided, 
+        # users can register_bare even if no password is provided,
         # in this case they will have to reset their password to login
         if fields.get(settings.passfield):
             fields[settings.passfield] = \
@@ -2381,7 +2381,7 @@ class Auth(object):
         if not fields.get(settings.userfield):
             raise ValueError('register_bare: ' +
                              'userfield not provided or invalid')
-        user = self.get_or_create_user(fields, login=False, get=False, 
+        user = self.get_or_create_user(fields, login=False, get=False,
                                        update_fields=self.settings.update_fields)
         if not user:
             # get or create did not create a user (it ignores duplicate records)
@@ -2732,7 +2732,7 @@ class Auth(object):
         # username and password at the first challenge).
         # Check if this user is signed up for two-factor authentication
         # If auth.settings.auth_two_factor_enabled it will enable two factor
-        # for all the app. Another way to anble two factor is that the user 
+        # for all the app. Another way to anble two factor is that the user
         # must be part of a group that is called auth.settings.two_factor_authentication_group
         if user and self.settings.auth_two_factor_enabled == True:
             session.auth_two_factor_enabled = True
@@ -2763,7 +2763,7 @@ class Auth(object):
                 session.auth_two_factor_tries_left = self.settings.auth_two_factor_tries_left
                 # Set the way we generate the code or we send the code. For example using SMS...
                 two_factor_methods = self.settings.two_factor_methods
-                
+
                 if two_factor_methods == []:
                     # TODO: Add some error checking to handle cases where email cannot be sent
                     self.settings.mailer.send(
@@ -2780,19 +2780,19 @@ class Auth(object):
                             pass
                         else:
                             break
-                        
+
             if form.accepts(request, session if self.csrf_prevention else None,
                             formname='login', dbio=False,
                             onvalidation=onvalidation,
                             hideerror=settings.hideerror):
                 accepted_form = True
-                
+
                 accepted_form = True
-                
+
                 '''
                 The lists is executed after form validation for each of the corresponding action.
                 For example, in your model:
-                
+
                 In your models copy and paste:
 
                 #Before define tables, we add some extra field to auth_user
@@ -2802,7 +2802,7 @@ class Auth(object):
 
                 OFFSET = 60 #Be sure is the same in your OTP Client
 
-                #Set session.auth_two_factor to None. Because the code is generated by external app. 
+                #Set session.auth_two_factor to None. Because the code is generated by external app.
                 # This will avoid to use the default setting and send a code by email.
                 def _set_two_factor(user, auth_two_factor):
                     return None
@@ -2823,10 +2823,10 @@ class Auth(object):
                 auth.messages.two_factor_comment = "Verify your OTP Client for the code."
                 auth.settings.two_factor_methods = [lambda user, auth_two_factor: _set_two_factor(user, auth_two_factor)]
                 auth.settings.two_factor_onvalidation = [lambda user, otp: verify_otp(user, otp)]
-                
+
                 '''
                 if self.settings.two_factor_onvalidation != []:
-                    
+
                     for two_factor_onvalidation in self.settings.two_factor_onvalidation:
                         try:
                             session.auth_two_factor = two_factor_onvalidation(session.auth_two_factor_user, form.vars['authentication_code'])
@@ -2834,7 +2834,7 @@ class Auth(object):
                             pass
                         else:
                             break
-                        
+
                 if form.vars['authentication_code'] == str(session.auth_two_factor):
                     # Handle the case when the two-factor form has been successfully validated
                     # and the user was previously stored (the current user should be None because
@@ -2994,7 +2994,7 @@ class Auth(object):
         except:
             pass
 
-        if self.settings.register_verify_password:            
+        if self.settings.register_verify_password:
             if self.settings.register_fields is None:
                 self.settings.register_fields = [f.name for f in table_user if f.writable]
                 k = self.settings.register_fields.index("password")
@@ -3005,7 +3005,7 @@ class Auth(object):
                                            error_message=self.messages.mismatched_password),
                       label=current.T("Confirm Password"))]
         else:
-            extra_fields = []        
+            extra_fields = []
         form = SQLFORM(table_user,
                        fields=self.settings.register_fields,
                        hidden=dict(_next=next),
@@ -3382,7 +3382,7 @@ class Auth(object):
 
         if form.process().accepted:
             emails = re.compile('[^\s\'"@<>,;:]+\@[^\s\'"@<>,;:]+').findall(form.vars.emails)
-            # send the invitations            
+            # send the invitations
             emails_sent = []
             emails_fail = []
             emails_exist = []
@@ -3403,7 +3403,7 @@ class Auth(object):
 
     def manage_tokens(self):
         if not self.user:
-            redirect(self.settings.login_url)        
+            redirect(self.settings.login_url)
         table_token =self.table_token()
         table_token.user_id.writable = False
         table_token.user_id.default = self.user.id
diff --git a/gluon/validators.py b/gluon/validators.py
index d6f03efd..24510607 100644
--- a/gluon/validators.py
+++ b/gluon/validators.py
@@ -3367,7 +3367,7 @@ class IS_IPV4(Validator):
                     (number == self.localhost)):
                     ok = False
             if not (self.is_private is None or self.is_private ==
-                    (sum([private_number[0] <= number <= private_number[1] 
+                    (sum([private_number[0] <= number <= private_number[1]
                           for private_number in self.private]) > 0)):
                     ok = False
             if not (self.is_automatic is None or self.is_automatic ==