Merge branch 'master' into typos2

applications/examples/views/default/download.html

@@ -81,7 +81,7 @@
 
 <h3 id="license">License</h3>
 <p>Web2py code is released under <a href="http://www.gnu.org/licenses/lgpl.html">LGPLv3 License</a>. This license does not extend to third party libraries distributed with web2py (which can be MIT, BSD or Apache type licenses) nor does it extend to applications built with web2py (under the terms of the LGPL.</p>
-<p>Applications built with web2py can be released under any license the author wishes as long they do not contain web2py code. They can link unmodified web2py libraries and they can be distributed with official web2py binaries. In particular web2py applications can be distributed in closed source. The admin interface provides a button to byte-code compile.</p>
+<p>Applications built with web2py can be released under any license the author wishes as long as they do not contain web2py code. They can link unmodified web2py libraries and they can be distributed with official web2py binaries. In particular web2py applications can be distributed in closed source. The admin interface provides a button to byte-code compile.</p>
 <p>It is fine to distribute web2py (source or compiled) with your applications as long as you make it clear in the license where your application ends and web2py starts.</p>
 <p>web2py is copyrighted by Massimo Di Pierro. The web2py trademark is owned by Massimo Di Pierro.</p>
 <a class="btn btn-small rounded" href="{{=URL('license')}}">read more</a>

applications/examples/views/default/examples.html

@@ -50,10 +50,10 @@ def hello3():
 
     <b>and view: simple_examples/hello3.html</b>
     {{=CODE(open(os.path.join(request.folder,'views/simple_examples/hello3.html'),'r').read(),language='html',link=URL('global','vars'),_class='boxCode')}}
-    <p>If you return a dictionary, the variables defined in the dictionery are visible to the view (template).
+    <p>If you return a dictionary, the variables defined in the dictionary are visible to the view (template).
     <br/>Try it here: <a class="btn" href="/{{=request.application}}/simple_examples/hello3.html">hello3</a></p>
 
-    <p>Actions can also be be rendered in other formsts like JSON, <a href="/{{=request.application}}/simple_examples/hello3.json">hello3.json</a>, and XML, <a href="/{{=request.application}}/simple_examples/hello3.xml">hello3.xml</a></p>
+    <p>Actions can also be be rendered in other forms like JSON, <a href="/{{=request.application}}/simple_examples/hello3.json">hello3.json</a>, and XML, <a href="/{{=request.application}}/simple_examples/hello3.xml">hello3.xml</a></p>
 
     <h3>Example {{=c}}{{c+=1}}</h3><b>In controller: simple_examples.py</b>
     {{=CODE("""
@@ -69,7 +69,7 @@ def hello4():
 def hello5():
     return HTML(BODY(H1(T('Hello World'),_style="color: red;"))).xml() # .xml to serialize
     """.strip(),language='web2py',link=URL('global','vars'),_class='boxCode')}}
-    <p>You can also generate HTML using helper objects HTML, BODY, H1, etc. Each of these tags is a class and the views know how to render the corresponding objects. The method .xml() serializes them and produce html/xml code for the page.
+    <p>You can also generate HTML using helper objects HTML, BODY, H1, etc. Each of these tags is a class and the views know how to render the corresponding objects. The method .xml() serializes them and produces html/xml code for the page.
     Each tag, DIV for example, takes three types of arguments:</p>
     <ul>
     <li>unnamed arguments, they correspond to nested tags</li>
@@ -201,11 +201,11 @@ def ajaxwiki_onclick():
     <h3>Example {{=c}}{{c+=1}}</h3><b>In controller: session_examples.py </b>
     {{=CODE("""
 def counter():
-    session.counter = (sesstion.counter or 0) + 1
+    session.counter = (session.counter or 0) + 1
     return dict(counter=session.counter)
     """.strip(),language='web2py',link=URL('global','vars'),_class='boxCode')}}<b>and view: session_examples/counter.html</b>
     {{=CODE(open(os.path.join(request.folder,'views/session_examples/counter.html'),'r').read(),language='html',link=URL('global','vars'),_class='boxCode')}}
-    <p>Click to count. The session.counter is persistent for this user and application. Every applicaiton within the system has its own separate session management.
+    <p>Click to count. The session.counter is persistent for this user and application. Every application within the system has its own separate session management.
     <br/>Try it here: <a class="btn" href="/{{=request.application}}/session_examples/counter">counter</a></p>
 
     <h2 id="template_examples">Template Examples</h2>

applications/examples/views/default/who.html

@@ -150,6 +150,7 @@
 <li><a href="http://www.python.org">Python</a> created by Guido van Rossum.</li>
 <li>Rocket Web Server developed by Timothy Farrell.</li>
 <li><a href="http://codemirror.net/">CodeMirror</a></li>
+
 <li><a href="http://pyrtf.sourceforge.net/">PyRTF</a> developed by Simon Cusack and revised by Grant Edwards</li>
 <li><a href="http://www.dalkescientific.com/Python/PyRSS2Gen.html">PyRSS2Gen</a> developed by Dalke Scientific Software</li>
 <li><a href="http://www.feedparser.org/">feedparser</a> developed by Mark Pilgrim</li>

gluon/contrib/login_methods/ldap_auth.py

@@ -170,7 +170,9 @@ def ldap_auth(server='ldap',
     is "error" and can be set to error, warning, info, debug.
     """
     logger = logging.getLogger('web2py.auth.ldap_auth')
-    if logging_level == 'error':
+    if isinstance(logging_level, int):
+        logger.setLevel(logging_level)
+    elif logging_level == 'error':
         logger.setLevel(logging.ERROR)
     elif logging_level == 'warning':
         logger.setLevel(logging.WARNING)
@@ -398,17 +400,25 @@ def ldap_auth(server='ldap',
             if manage_user:
                 logger.info('[%s] Manage user data' % str(username))
                 try:
+                    user_firstname = result[user_firstname_attrib][0]
                     if user_firstname_part is not None:
-                        store_user_firstname = result[user_firstname_attrib][0].split(' ', 1)[user_firstname_part]
+                        store_user_firstname = user_firstname.split(
+                            b' ' if isinstance(user_firstname, bytes) else ' ',
+                            1
+                        )[user_firstname_part]
                     else:
-                        store_user_firstname = result[user_firstname_attrib][0]
+                        store_user_firstname = user_firstname
                 except KeyError as e:
                     store_user_firstname = None
                 try:
+                    user_lastname = result[user_lastname_attrib][0]
                     if user_lastname_part is not None:
-                        store_user_lastname = result[user_lastname_attrib][0].split(' ', 1)[user_lastname_part]
+                        store_user_lastname = user_lastname.split(
+                            b' ' if isinstance(user_lastname, bytes) else ' ',
+                            1
+                        )[user_lastname_part]
                     else:
-                        store_user_lastname = result[user_lastname_attrib][0]
+                        store_user_lastname = user_lastname
                 except KeyError as e:
                     store_user_lastname = None
                 try:

gluon/contrib/redis_cache.py

@@ -10,7 +10,7 @@ except:
 import time
 import re
 import logging
-import thread
+from threading import Lock
 import random
 from gluon import current
 from gluon.cache import CacheAbstract
@@ -19,7 +19,7 @@ from gluon.contrib.redis_utils import register_release_lock, RConnectionError
 
 logger = logging.getLogger("web2py.cache.redis")
 
-locker = thread.allocate_lock()
+locker = Lock()
 
 
 def RedisCache(redis_conn=None, debug=False, with_lock=False, fail_gracefully=False, db=None):

gluon/contrib/redis_session.py

@@ -8,15 +8,16 @@ Redis-backed sessions
 """
 
 import logging
-import thread
+from threading import Lock
 from gluon import current
 from gluon.storage import Storage
 from gluon.contrib.redis_utils import acquire_lock, release_lock
 from gluon.contrib.redis_utils import register_release_lock
+from gluon._compat import to_bytes
 
 logger = logging.getLogger("web2py.session.redis")
 
-locker = thread.allocate_lock()
+locker = Lock()
 
 
 def RedisSession(redis_conn, session_expiry=False, with_lock=False, db=None):
@@ -43,7 +44,7 @@ def RedisSession(redis_conn, session_expiry=False, with_lock=False, db=None):
     try:
         instance_name = 'redis_instance_' + current.request.application
         if not hasattr(RedisSession, instance_name):
-            setattr(RedisSession, instance_name, 
+            setattr(RedisSession, instance_name,
                     RedisClient(redis_conn, session_expiry=session_expiry, with_lock=with_lock))
         return getattr(RedisSession, instance_name)
     finally:
@@ -185,8 +186,8 @@ class MockQuery(object):
             if rtn:
                 if self.unique_key:
                     # make sure the id and unique_key are correct
-                    if rtn['unique_key'] == self.unique_key:
-                        rtn['update_record'] = self.update  # update record support
+                    if rtn[b'unique_key'] == to_bytes(self.unique_key):
+                        rtn[b'update_record'] = self.update  # update record support
                     else:
                         rtn = None
             return [Storage(rtn)] if rtn else []

gluon/contrib/redis_utils.py

@@ -11,7 +11,7 @@ to ensure compatibility with another - similar - library
 """
 
 import logging
-import thread
+from threading import Lock
 import time
 from gluon import current
 
@@ -26,7 +26,7 @@ except ImportError:
     raise RuntimeError('Needs redis library to work')
 
 
-locker = thread.allocate_lock()
+locker = Lock()
 
 
 def RConn(*args, **vars):

gluon/globals.py

@@ -14,7 +14,7 @@ Contains the classes for the global used variables:
 
 """
 from gluon._compat import pickle, StringIO, copyreg, Cookie, urlparse, PY2, iteritems, to_unicode, to_native, \
-    unicodeT, long, hashlib_md5, urllib_quote
+    to_bytes, unicodeT, long, hashlib_md5, urllib_quote
 from gluon.storage import Storage, List
 from gluon.streamer import streamer, stream_file_or_304_or_206, DEFAULT_CHUNK_SIZE
 from gluon.contenttype import contenttype
@@ -225,7 +225,7 @@ class Request(Storage):
                 # instead of load.
                 # This line can be simplified to json_vars = json_parser.load(body)
                 # if and when we drop support for python versions under 3.6
-                json_vars = json_parser.loads(to_native(body.read())) 
+                json_vars = json_parser.loads(to_native(body.read()))
             except:
                 # incoherent request bodies can still be parsed "ad-hoc"
                 json_vars = {}
@@ -344,8 +344,8 @@ class Request(Storage):
         user_agent = Storage(user_agent)
         user_agent.is_mobile = 'Mobile' in http_user_agent
         user_agent.is_tablet = 'Tablet' in http_user_agent
-        session._user_agent = user_agent 
-        
+        session._user_agent = user_agent
+
         return user_agent
 
     def requires_https(self):
@@ -485,12 +485,12 @@ class Response(Storage):
         #
         # We will only minify and concat adjacent internal files as there's
         # no way to know if changing the order with which the files are apppended
-        # will break things since the order matters in both CSS and JS and 
+        # will break things since the order matters in both CSS and JS and
         # internal files may be interleaved with external ones.
         files = []
         # For the adjacent list we're going to use storage List to both distinguish
         # from the regular list and so we can add attributes
-        internal = List()  
+        internal = List()
         internal.has_js = False
         internal.has_css = False
         done = set() # to remove duplicates
@@ -513,7 +513,7 @@ class Response(Storage):
             if item.endswith('.js'):
                 internal.has_js = True
             if item.endswith('.css'):
-                internal.has_css = True            
+                internal.has_css = True
         if internal:
             files.append(internal)
 
@@ -537,7 +537,7 @@ class Response(Storage):
                                             time_expire)
                     else:
                         files[i] = call_minify()
-        
+
         def static_map(s, item):
             if isinstance(item, str):
                 f = item.lower().split('?')[0]
@@ -967,7 +967,7 @@ class Session(Storage):
                     if row:
                         # rows[0].update_record(locked=True)
                         # Unpickle the data
-                        session_data = pickle.loads(row.session_data)
+                        session_data = pickle.loads(row[b'session_data'])
                         self.update(session_data)
                         response.session_new = False
                     else:
@@ -1049,7 +1049,7 @@ class Session(Storage):
             if record_id.isdigit() and long(record_id) > 0:
                 new_unique_key = web2py_uuid()
                 row = table(record_id)
-                if row and row.unique_key == unique_key:
+                if row and row[b'unique_key'] == to_bytes(unique_key):
                     table._db(table.id == record_id).update(unique_key=new_unique_key)
                 else:
                     record_id = None
@@ -1167,7 +1167,7 @@ class Session(Storage):
                              compression_level=compression_level)
         rcookies = response.cookies
         rcookies.pop(name, None)
-        rcookies[name] = value
+        rcookies[name] = to_native(value)
         rcookies[name]['path'] = '/'
         expires = response.session_cookie_expires
         if isinstance(expires, datetime.datetime):

gluon/tests/test_tools.py

@@ -17,7 +17,7 @@ DEFAULT_URI = os.getenv('DB', 'sqlite:memory')
 from gluon.dal import DAL, Field
 from pydal.objects import Table
 from gluon import tools
-from gluon.tools import Auth, Mail, Recaptcha, Recaptcha2, prettydate, Expose
+from gluon.tools import Auth, Mail, Recaptcha2, prettydate, Expose
 from gluon._compat import PY2
 from gluon.globals import Request, Response, Session
 from gluon.storage import Storage
@@ -208,14 +208,6 @@ class TestMail(unittest.TestCase):
         self.assertTrue('Content-Id: <trololo>' in message.payload)
 
 
-# class TestRecaptcha(unittest.TestCase):
-#     def test_Recaptcha(self):
-#         from html import FORM
-#         form = FORM(Recaptcha(public_key='public_key', private_key='private_key'))
-#         self.assertEqual(form.xml(),
-#                          '<form action="#" enctype="multipart/form-data" method="post"><div id="recaptcha"><script><!--\nvar RecaptchaOptions = {};\n//--></script><script src="http://www.google.com/recaptcha/api/challenge?k=public_key" type="text/javascript"></script><noscript><iframe frameborder="0" height="300" src="http://www.google.com/recaptcha/api/noscript?k=public_key" width="500"></iframe><br /><input name="recaptcha_response_field" type="hidden" value="manual_challenge" /></noscript></div></form>')
-#
-#
 # class TestRecaptcha2(unittest.TestCase):
 #     def test_Recaptcha2(self):
 #         from html import FORM
@@ -248,7 +240,6 @@ class TestAuthJWT(unittest.TestCase):
                                          self.user_data['password'])[0]))
         self.jwtauth = AuthJWT(self.auth, secret_key='secret', verify_expiration=True)
 
-
     def test_jwt_token_manager(self):
         import gluon.serializers
         self.request.vars.update(self.user_data)
@@ -260,7 +251,6 @@ class TestAuthJWT(unittest.TestCase):
         self.token = self.jwtauth.jwt_token_manager()
         self.assertIsNotNone(self.token)
 
-
     def test_allows_jwt(self):
         import gluon.serializers
         self.request.vars.update(self.user_data)
@@ -270,11 +260,13 @@ class TestAuthJWT(unittest.TestCase):
         del self.request.vars['password']
         self.token = self.jwtauth.jwt_token_manager()
         self.request.vars._token = gluon.serializers.json_parser.loads(self.token)['token']
+
         @self.jwtauth.allows_jwt()
         def optional_auth():
             self.assertEqual(self.user_data['username'], self.auth.user.username)
         optional_auth()
 
+
 @unittest.skipIf(IS_IMAP, "TODO: Imap raises 'Connection refused'")
 # class TestAuth(unittest.TestCase):
 #
@@ -495,8 +487,6 @@ class TestAuthJWT(unittest.TestCase):
 #     #     impersonate_form = auth.impersonate(user_id=omer_id)
 #     #     self.assertTrue(auth.is_impersonating())
 #     #     self.assertEqual(impersonate_form, 'test')
-
-
 class TestAuth(unittest.TestCase):
 
     def myassertRaisesRegex(self, *args, **kwargs):
@@ -904,7 +894,7 @@ class TestAuth(unittest.TestCase):
         self.assertEqual(count_log_event_test_after, count_log_event_test_before)
 
     def test_add_membership(self):
-        user = self.db(self.db.auth_user.username == 'bart').select().first() # bypass login_bare()
+        user = self.db(self.db.auth_user.username == 'bart').select().first()  # bypass login_bare()
         user_id = user.id
         role_name = 'test_add_membership_group'
         group_id = self.auth.add_group(role_name)
@@ -1174,6 +1164,7 @@ class TestToolsFunctions(unittest.TestCase):
 
 pjoin = os.path.join
 
+
 def have_symlinks():
     return os.name == 'posix'
 
@@ -1181,18 +1172,20 @@ def have_symlinks():
 class Test_Expose__in_base(unittest.TestCase):
 
     def test_in_base(self):
-        are_under = [ # (sub, base)
+        are_under = [
+            # (sub, base)
             ('/foo/bar', '/foo'),
             ('/foo', '/foo'),
             ('/foo', '/'),
             ('/', '/'),
         ]
         for sub, base in are_under:
-            self.assertTrue( Expose._Expose__in_base(subdir=sub, basedir=base, sep='/'),
-                             '%s is not under %s' % (sub, base) )
+            self.assertTrue(Expose._Expose__in_base(subdir=sub, basedir=base, sep='/'),
+                            '%s is not under %s' % (sub, base))
 
     def test_not_in_base(self):
-        are_not_under = [ # (sub, base)
+        are_not_under = [
+            # (sub, base)
             ('/foobar', '/foo'),
             ('/foo', '/foo/bar'),
             ('/bar', '/foo'),
@@ -1200,8 +1193,8 @@ class Test_Expose__in_base(unittest.TestCase):
             ('/', '/x'),
         ]
         for sub, base in are_not_under:
-            self.assertFalse( Expose._Expose__in_base(subdir=sub, basedir=base, sep='/'),
-                              '%s should not be under %s' % (sub, base) )
+            self.assertFalse(Expose._Expose__in_base(subdir=sub, basedir=base, sep='/'),
+                             '%s should not be under %s' % (sub, base))
 
 
 class TestExpose(unittest.TestCase):
@@ -1237,7 +1230,7 @@ class TestExpose(unittest.TestCase):
         shutil.rmtree(self.base_dir)
 
     def make_dirs(self):
-        """setup direcotry strucutre"""
+        """setup directory structure"""
         for d in (['inside'],
                   ['inside', 'dir1'],
                   ['inside', 'dir2'],
@@ -1257,7 +1250,7 @@ class TestExpose(unittest.TestCase):
             f.write('README content')
 
     def make_symlinks(self):
-        """setup extenstion for posix systems"""
+        """setup extension for posix systems"""
         # inside links
         os.symlink(
             pjoin(self.base_dir, 'inside', 'dir1'),

gluon/tools.py

@@ -52,7 +52,7 @@ import gluon.serializers as serializers
 Table = DAL.Table
 Field = DAL.Field
 
-__all__ = ['Mail', 'Auth', 'Recaptcha', 'Recaptcha2', 'Crud', 'Service', 'Wiki',
+__all__ = ['Mail', 'Auth', 'Recaptcha2', 'Crud', 'Service', 'Wiki',
            'PluginManager', 'fetch', 'geocode', 'reverse_geocode', 'prettydate']
 
 # mind there are two loggers here (logger and crud.settings.logger)!
@@ -826,149 +826,6 @@ class Mail(object):
         return True
 
 
-class Recaptcha(DIV):
-
-    """
-    Examples:
-        Use as::
-
-            form = FORM(Recaptcha(public_key='...', private_key='...'))
-
-        or::
-
-            form = SQLFORM(...)
-            form.append(Recaptcha(public_key='...', private_key='...'))
-
-    """
-
-    API_SSL_SERVER = 'https://www.google.com/recaptcha/api'
-    API_SERVER = 'http://www.google.com/recaptcha/api'
-    VERIFY_SERVER = 'http://www.google.com/recaptcha/api/verify'
-
-    def __init__(self,
-                 request=None,
-                 public_key='',
-                 private_key='',
-                 use_ssl=False,
-                 error=None,
-                 error_message='invalid',
-                 label='Verify:',
-                 options='',
-                 comment='',
-                 ajax=False
-                 ):
-        request = request or current.request
-        self.request_vars = request and request.vars or current.request.vars
-        self.remote_addr = request.env.remote_addr
-        self.public_key = public_key
-        self.private_key = private_key
-        self.use_ssl = use_ssl
-        self.error = error
-        self.errors = Storage()
-        self.error_message = error_message
-        self.components = []
-        self.attributes = {}
-        self.label = label
-        self.options = options
-        self.comment = comment
-        self.ajax = ajax
-
-    def _validate(self):
-
-        # for local testing:
-
-        recaptcha_challenge_field = \
-            self.request_vars.recaptcha_challenge_field
-        recaptcha_response_field = \
-            self.request_vars.recaptcha_response_field
-        private_key = self.private_key
-        remoteip = self.remote_addr
-        if not (recaptcha_response_field and recaptcha_challenge_field
-                and len(recaptcha_response_field)
-                and len(recaptcha_challenge_field)):
-            self.errors['captcha'] = self.error_message
-            return False
-        params = urlencode({
-            'privatekey': private_key,
-            'remoteip': remoteip,
-            'challenge': recaptcha_challenge_field,
-            'response': recaptcha_response_field,
-        })
-        request = urllib2.Request(
-            url=self.VERIFY_SERVER,
-            data=params,
-            headers={'Content-type': 'application/x-www-form-urlencoded',
-                     'User-agent': 'reCAPTCHA Python'})
-        httpresp = urllib2.urlopen(request)
-        return_values = httpresp.read().splitlines()
-        httpresp.close()
-        return_code = return_values[0]
-        if return_code == 'true':
-            del self.request_vars.recaptcha_challenge_field
-            del self.request_vars.recaptcha_response_field
-            self.request_vars.captcha = ''
-            return True
-        else:
-            # In case we get an error code, store it so we can get an error message
-            # from the /api/challenge URL as described in the reCAPTCHA api docs.
-            self.error = return_values[1]
-            self.errors['captcha'] = self.error_message
-            return False
-
-    def xml(self):
-        public_key = self.public_key
-        use_ssl = self.use_ssl
-        error_param = ''
-        if self.error:
-            error_param = '&error=%s' % self.error
-        if use_ssl:
-            server = self.API_SSL_SERVER
-        else:
-            server = self.API_SERVER
-        if not self.ajax:
-            captcha = DIV(
-                SCRIPT("var RecaptchaOptions = {%s};" % self.options),
-                SCRIPT(_type="text/javascript",
-                       _src="%s/challenge?k=%s%s" % (server, public_key, error_param)),
-                TAG.noscript(
-                    IFRAME(
-                        _src="%s/noscript?k=%s%s" % (
-                            server, public_key, error_param),
-                        _height="300", _width="500", _frameborder="0"), BR(),
-                    INPUT(
-                        _type='hidden', _name='recaptcha_response_field',
-                        _value='manual_challenge')), _id='recaptcha')
-
-        else:  # use Google's ajax interface, needed for LOADed components
-
-            url_recaptcha_js = "%s/js/recaptcha_ajax.js" % server
-            RecaptchaOptions = "var RecaptchaOptions = {%s}" % self.options
-            script = """%(options)s;
-            jQuery.getScript('%(url)s',function() {
-                Recaptcha.create('%(public_key)s',
-                    'recaptcha',jQuery.extend(RecaptchaOptions,{'callback':Recaptcha.focus_response_field}))
-                }) """ % ({'options': RecaptchaOptions, 'url': url_recaptcha_js, 'public_key': public_key})
-            captcha = DIV(
-                SCRIPT(
-                    script,
-                    _type="text/javascript",
-                ),
-                TAG.noscript(
-                    IFRAME(
-                        _src="%s/noscript?k=%s%s" % (
-                            server, public_key, error_param),
-                        _height="300", _width="500", _frameborder="0"), BR(),
-                    INPUT(
-                        _type='hidden', _name='recaptcha_response_field',
-                        _value='manual_challenge')), _id='recaptcha')
-
-        if not self.errors.captcha:
-            return XML(captcha).xml()
-        else:
-            captcha.append(DIV(self.errors['captcha'], _class='error'))
-            return XML(captcha).xml()
-
-
 class Recaptcha2(DIV):
     """
     Experimental:

gluon/utils.py

@@ -207,6 +207,7 @@ def secure_dumps(data, encryption_key, hash_key=None, compression_level=None):
 
 
 def secure_loads(data, encryption_key, hash_key=None, compression_level=None):
+    data = to_bytes(data)
     components = data.count(b':')
     if components == 1:
         return secure_loads_deprecated(data, encryption_key, hash_key, compression_level)