faster non forms

Tracking latest pyDAL changes

.gitignore

@@ -59,3 +59,4 @@ HOWTO-web2py-devel
 *.sublime-workspace
 .idea/*
 site-packages/
+logs/

CHANGELOG

@@ -1,5 +1,6 @@
-## 2.13.1
+## 2.13.1-2
 
+- fixed a security issue in request_reset_password
 - added fabfile.py
 - fixed oauth2 renew token, thanks dokime7
 - fixed add_membership, del_membership, add_membership IntegrityError (when auth.enable_record_versioning) 

Makefile

@@ -11,7 +11,7 @@ clean:
 	find ./ -name '*.rej' -exec rm -f {} \; 
 	find ./ -name '#*' -exec rm -f {} \;
 	find ./ -name 'Thumbs.db' -exec rm -f {} \; 
-	find ./gluon/ -name '.*' -exec rm -f {} \;
+	# find ./gluon/ -name '.*' -exec rm -f {} \;
 	find ./gluon/ -name '*class' -exec rm -f {} \; 
 	find ./applications/admin/ -name '.*' -exec rm -f {} \; 
 	find ./applications/examples/ -name '.*' -exec rm -f {} \; 
@@ -32,7 +32,7 @@ update:
 	echo "remember that pymysql was tweaked"
 src:
 	### Use semantic versioning
-	echo 'Version 2.13.1-stable+timestamp.'`date +%Y.%m.%d.%H.%M.%S` > VERSION
+	echo 'Version 2.13.4-stable+timestamp.'`date +%Y.%m.%d.%H.%M.%S` > VERSION
 	### rm -f all junk files
 	make clean
 	### clean up baisc apps

VERSION

@@ -1 +1 @@
-Version 2.12.3-stable+timestamp.2015.08.18.19.14.07
+Version 2.13.3-stable+timestamp.2015.12.24.08.08.22

applications/admin/controllers/appadmin.py

@@ -576,7 +576,7 @@ def bg_graph_model():
             meta_graphmodel = dict(group=request.application, color='#ECECEC')
 
         group = meta_graphmodel['group'].replace(' ', '')
-        if not subgraphs.has_key(group):
+        if group not in subgraphs:
             subgraphs[group] = dict(meta=meta_graphmodel, tables=[])
         subgraphs[group]['tables'].append(tablename)
 

applications/admin/views/appadmin.html

@@ -155,8 +155,8 @@
       {{=T.M("Cache contains items up to **%(hours)02d** %%{hour(hours)} **%(min)02d** %%{minute(min)} **%(sec)02d** %%{second(sec)} old.",
               dict(hours=total['oldest'][0], min=total['oldest'][1], sec=total['oldest'][2]))}}
     </p>
-    {{=BUTTON(T('Cache Keys'), _onclick='jQuery("#all_keys").toggle().toggleClass( "hidden" );')}}
-    <div class="hidden" id="all_keys">
+    {{=BUTTON(T('Cache Keys'), _onclick='jQuery("#all_keys").toggle().toggleClass( "w2p_hidden" );')}}
+    <div class="w2p_hidden" id="all_keys">
       {{=total['keys']}}
     </div>
     <br />
@@ -183,8 +183,8 @@
       {{=T.M("RAM contains items up to **%(hours)02d** %%{hour(hours)} **%(min)02d** %%{minute(min)} **%(sec)02d** %%{second(sec)} old.",
               dict(hours=ram['oldest'][0], min=ram['oldest'][1], sec=ram['oldest'][2]))}}
     </p>
-    {{=BUTTON(T('RAM Cache Keys'), _onclick='jQuery("#ram_keys").toggle().toggleClass( "hidden" );')}}
-    <div class="hidden" id="ram_keys">
+    {{=BUTTON(T('RAM Cache Keys'), _onclick='jQuery("#ram_keys").toggle().toggleClass( "w2p_hidden" );')}}
+    <div class="w2p_hidden" id="ram_keys">
       {{=ram['keys']}}
     </div>
     <br />
@@ -212,8 +212,8 @@
       {{=T.M("DISK contains items up to **%(hours)02d** %%{hour(hours)} **%(min)02d** %%{minute(min)} **%(sec)02d** %%{second(sec)} old.",
                 dict(hours=disk['oldest'][0], min=disk['oldest'][1], sec=disk['oldest'][2]))}}
       </p>
-      {{=BUTTON(T('Disk Cache Keys'), _onclick='jQuery("#disk_keys").toggle().toggleClass( "hidden" );')}}
-      <div class="hidden" id="disk_keys">
+      {{=BUTTON(T('Disk Cache Keys'), _onclick='jQuery("#disk_keys").toggle().toggleClass( "w2p_hidden" );')}}
+      <div class="w2p_hidden" id="disk_keys">
       {{=disk['keys']}}
       </div>
       <br />
@@ -249,8 +249,8 @@
           <li><a href="{{=URL('appadmin', 'bg_graph_model', args=['png'])}}">png</a></li>
           <li><a href="{{=URL('appadmin', 'bg_graph_model', args=['svg'])}}">svg</a></li>
           <li><a href="{{=URL('appadmin', 'bg_graph_model', args=['pdf'])}}">pdf</a></li>
-          <li><a href="{{=URL('appadmin', 'bg_graph_model', args=['ps'])}}">ps</a></li>          
-          <li><a href="{{=URL('appadmin', 'bg_graph_model', args=['dot'])}}">dot</a></li>                                   
+          <li><a href="{{=URL('appadmin', 'bg_graph_model', args=['ps'])}}">ps</a></li>
+          <li><a href="{{=URL('appadmin', 'bg_graph_model', args=['dot'])}}">dot</a></li>
       </ul>
     </div>
     <br />

applications/admin/views/default/design.html

@@ -207,7 +207,7 @@ for c in controllers: controller_functions+=[c[:-3]+'/%s.html'%x for x in functi
                 {{=peekfile('views',c, dict(id=id))}}
             </span>
             <span class="extras celled celled-one">
-                {{if extend.has_key(c):}}{{=T("extends")}} <b>{{=extend[c]}}</b> {{pass}}
+                {{if c in extend:}}{{=T("extends")}} <b>{{=extend[c]}}</b> {{pass}}
                 {{if include[c]:}}{{=T("includes")}} {{pass}}{{=XML(', '.join([B(f).xml() for f in include[c]]))}}
             </span>
         </li>

applications/admin/views/default/plugin.html

@@ -144,7 +144,7 @@ for c in controllers: controller_functions+=[c[:-3]+'/%s.html'%x for x in functi
 				{{=peekfile('views',c)}}
 			</span>
 			<span class="extras celled">
-				{{if extend.has_key(c):}}{{=T("extends")}} <b>{{=extend[c]}}</b> {{pass}}
+				{{if c in extend:}}{{=T("extends")}} <b>{{=extend[c]}}</b> {{pass}}
 				{{if include[c]:}}{{=T("includes")}} {{pass}}{{=XML(', '.join([B(f).xml() for f in include[c]]))}}
 			</span>
 		</li>

applications/examples/controllers/appadmin.py

@@ -576,7 +576,7 @@ def bg_graph_model():
             meta_graphmodel = dict(group=request.application, color='#ECECEC')
 
         group = meta_graphmodel['group'].replace(' ', '')
-        if not subgraphs.has_key(group):
+        if group not in subgraphs:
             subgraphs[group] = dict(meta=meta_graphmodel, tables=[])
         subgraphs[group]['tables'].append(tablename)
 

applications/examples/views/appadmin.html

@@ -155,8 +155,8 @@
       {{=T.M("Cache contains items up to **%(hours)02d** %%{hour(hours)} **%(min)02d** %%{minute(min)} **%(sec)02d** %%{second(sec)} old.",
               dict(hours=total['oldest'][0], min=total['oldest'][1], sec=total['oldest'][2]))}}
     </p>
-    {{=BUTTON(T('Cache Keys'), _onclick='jQuery("#all_keys").toggle().toggleClass( "hidden" );')}}
-    <div class="hidden" id="all_keys">
+    {{=BUTTON(T('Cache Keys'), _onclick='jQuery("#all_keys").toggle().toggleClass( "w2p_hidden" );')}}
+    <div class="w2p_hidden" id="all_keys">
       {{=total['keys']}}
     </div>
     <br />
@@ -183,8 +183,8 @@
       {{=T.M("RAM contains items up to **%(hours)02d** %%{hour(hours)} **%(min)02d** %%{minute(min)} **%(sec)02d** %%{second(sec)} old.",
               dict(hours=ram['oldest'][0], min=ram['oldest'][1], sec=ram['oldest'][2]))}}
     </p>
-    {{=BUTTON(T('RAM Cache Keys'), _onclick='jQuery("#ram_keys").toggle().toggleClass( "hidden" );')}}
-    <div class="hidden" id="ram_keys">
+    {{=BUTTON(T('RAM Cache Keys'), _onclick='jQuery("#ram_keys").toggle().toggleClass( "w2p_hidden" );')}}
+    <div class="w2p_hidden" id="ram_keys">
       {{=ram['keys']}}
     </div>
     <br />
@@ -212,8 +212,8 @@
       {{=T.M("DISK contains items up to **%(hours)02d** %%{hour(hours)} **%(min)02d** %%{minute(min)} **%(sec)02d** %%{second(sec)} old.",
                 dict(hours=disk['oldest'][0], min=disk['oldest'][1], sec=disk['oldest'][2]))}}
       </p>
-      {{=BUTTON(T('Disk Cache Keys'), _onclick='jQuery("#disk_keys").toggle().toggleClass( "hidden" );')}}
-      <div class="hidden" id="disk_keys">
+      {{=BUTTON(T('Disk Cache Keys'), _onclick='jQuery("#disk_keys").toggle().toggleClass( "w2p_hidden" );')}}
+      <div class="w2p_hidden" id="disk_keys">
       {{=disk['keys']}}
       </div>
       <br />
@@ -249,8 +249,8 @@
           <li><a href="{{=URL('appadmin', 'bg_graph_model', args=['png'])}}">png</a></li>
           <li><a href="{{=URL('appadmin', 'bg_graph_model', args=['svg'])}}">svg</a></li>
           <li><a href="{{=URL('appadmin', 'bg_graph_model', args=['pdf'])}}">pdf</a></li>
-          <li><a href="{{=URL('appadmin', 'bg_graph_model', args=['ps'])}}">ps</a></li>          
-          <li><a href="{{=URL('appadmin', 'bg_graph_model', args=['dot'])}}">dot</a></li>                                   
+          <li><a href="{{=URL('appadmin', 'bg_graph_model', args=['ps'])}}">ps</a></li>
+          <li><a href="{{=URL('appadmin', 'bg_graph_model', args=['dot'])}}">dot</a></li>
       </ul>
     </div>
     <br />

applications/welcome/controllers/appadmin.py

@@ -576,7 +576,7 @@ def bg_graph_model():
             meta_graphmodel = dict(group=request.application, color='#ECECEC')
 
         group = meta_graphmodel['group'].replace(' ', '')
-        if not subgraphs.has_key(group):
+        if group not in subgraphs:
             subgraphs[group] = dict(meta=meta_graphmodel, tables=[])
         subgraphs[group]['tables'].append(tablename)
 

gluon/compileapp.py

@@ -476,11 +476,11 @@ def compile_views(folder, skip_failed_views=False):
             data = parse_template(fname, path)
         except Exception, e:
             if skip_failed_views:
-                failed_views.append(file)
+                failed_views.append(fname)
             else:
-                raise Exception("%s in %s" % (e, file))
+                raise Exception("%s in %s" % (e, fname))
         else:
-            filename = ('views/%s.py' % file).replace('/', '_').replace('\\', '_')
+            filename = ('views/%s.py' % fname).replace('/', '_').replace('\\', '_')
             filename = pjoin(folder, 'compiled', filename)
             write_file(filename, data)
             save_pyc(filename)

gluon/contrib/login_methods/ldap_auth.py

@@ -432,7 +432,8 @@ def ldap_auth(server='ldap',
                     # #############
                     fields = ['first_name', 'last_name']
                     user_in_db = db(db.auth_user.email == username)
-                update_or_insert_values = {f: update_or_insert_values[f] for f in fields}
+                    update_or_insert_values = dict(((f, update_or_insert_values[f]) for f in fields))
+
                 if user_in_db.count() > 0:
                     actual_values = user_in_db.select(*[db.auth_user[f] for f in fields]).first().as_dict()
                     if update_or_insert_values != actual_values:  # We don't update record if values are the same

gluon/contrib/web2py_jwt.py

@@ -1,304 +0,0 @@
-# -*- coding: utf-8 -*-
-import datetime
-import uuid
-import time
-from gluon.serializers import json_parser
-import base64
-import hmac
-import hashlib
-from gluon.storage import Storage
-from gluon.utils import web2py_uuid
-from gluon import current
-from gluon.http import HTTP
-
-
-class Web2pyJwt(object):
-
-    """
-    If left externally, this needs the usual "singleton" approach.
-    Given I (we) don't know if to include in auth yet, let's stick to basics.
-
-    Args:
-     - secret_key: the secret. Without salting, an attacker knowing this can impersonate
-                   any user
-     - algorithm : uses as they are in the JWT specs, HS256, HS384 or HS512 basically means
-                   signing with HMAC with a 256, 284 or 512bit hash
-     - verify_expiration : verifies the expiration checking the exp claim
-     - leeway: allow n seconds of skew when checking for token expiration
-     - expiration : how many seconds a token may be valid
-     - allow_refresh: enable the machinery to get a refreshed token passing a not-already-expired
-                      token
-     - refresh_expiration_delta: to avoid continous refresh of the token
-     - header_prefix : self-explanatory. "JWT" and "Bearer" seems to be the emerging standards
-     - jwt_add_header: a dict holding additional mappings to the header. by default only alg and typ are filled
-     - user_param: the name of the parameter holding the username when requesting a token. Can be useful, e.g, for
-                   email-based authentication, with "email" as a parameter
-     - pass_param: same as above, but for the password
-     - realm: self-explanatory
-     - salt: can be static or a function that takes the payload as an argument.
-             Example:
-             def mysalt(payload):
-                return payload['hmac_key'].split('-')[0]
-     - additional_payload: can be a dict to merge with the payload or a function that takes
-                           the payload as input and returns the modified payload
-                           Example:
-                           def myadditional_payload(payload):
-                               payload['my_name_is'] = 'bond,james bond'
-                               return payload
-     - before_authorization: can be a callable that takes the deserialized token (a dict) as input.
-                             Gets called right after signature verification but before the actual
-                             authorization takes place. You can raise with HTTP a proper error message
-                             Example:
-                             def mybefore_authorization(tokend):
-                                 if not tokend['my_name_is'] == 'bond,james bond':
-                                     raise HTTP(400, u'Invalid JWT my_name_is claim')
-     - max_header_length: check max length to avoid load()ing unusually large tokens (could mean crafted, e.g. in a DDoS.)
-
-    Basic Usage:
-    in models (or the controller needing it)
-
-        myjwt = Web2pyJwt('secret', auth)
-
-    in the controller issuing tokens
-
-        def login_and_take_token():
-            return myjwt.jwt_token_manager()
-
-    A call then to /app/controller/login_and_take_token/auth with username and password returns the token
-    A call to /app/controller/login_and_take_token/refresh with the original token returns the refreshed token
-
-    To protect a function with JWT
-
-        @myjwt.requires_jwt()
-        @auth.requires_login()
-        def protected():
-            return '%s$%s' % (request.now, auth.user_id)
-
-    """
-
-    def __init__(self, secret_key,
-                 auth,
-                 algorithm='HS256',
-                 verify_expiration=True,
-                 leeway=30,
-                 expiration=60 * 5,
-                 allow_refresh=True,
-                 refresh_expiration_delta=60 * 60,
-                 header_prefix='Bearer',
-                 jwt_add_header=None,
-                 user_param='username',
-                 pass_param='password',
-                 realm='Login required',
-                 salt=None,
-                 additional_payload=None,
-                 before_authorization=None,
-                 max_header_length=4*1024,
-                 ):
-        self.secret_key = secret_key
-        self.auth = auth
-        self.algorithm = algorithm
-        if self.algorithm not in ('HS256', 'HS384', 'HS512'):
-            raise NotImplementedError('Algoritm %s not allowed' % algorithm)
-        self.verify_expiration = verify_expiration
-        self.leeway = leeway
-        self.expiration = expiration
-        self.allow_refresh = allow_refresh
-        self.refresh_expiration_delta = refresh_expiration_delta
-        self.header_prefix = header_prefix
-        self.jwt_add_header = jwt_add_header or {}
-        base_header = {'alg': self.algorithm, 'typ': 'JWT'}
-        for k, v in self.jwt_add_header.iteritems():
-            base_header[k] = v
-        self.cached_b64h = self.jwt_b64e(json_parser.dumps(base_header))
-        digestmod_mapping = {
-            'HS256': hashlib.sha256,
-            'HS384': hashlib.sha384,
-            'HS512': hashlib.sha512
-        }
-        self.digestmod = digestmod_mapping[algorithm]
-        self.user_param = user_param
-        self.pass_param = pass_param
-        self.realm = realm
-        self.salt = salt
-        self.additional_payload = additional_payload
-        self.before_authorization = before_authorization
-        self.max_header_length = max_header_length
-        print 'initialized'
-
-    @staticmethod
-    def jwt_b64e(string):
-        if isinstance(string, unicode):
-            string = string.encode('uft-8', 'strict')
-        return base64.urlsafe_b64encode(string).strip(b'=')
-
-    @staticmethod
-    def jwt_b64d(string):
-        """base64 decodes a single bytestring (and is tolerant to getting
-        called with a unicode string).
-        The result is also a bytestring.
-        """
-        if isinstance(string, unicode):
-            string = string.encode('ascii', 'ignore')
-        return base64.urlsafe_b64decode(string + '=' * (-len(string) % 4))
-
-    def generate_token(self, payload):
-        secret = self.secret_key
-        if self.salt:
-            if callable(self.salt):
-                secret = "%s$%s" % (secret, self.salt(payload))
-            else:
-                secret = "%s$%s" % (secret, self.salt)
-            if isinstance(secret, unicode):
-                secret = secret.encode('ascii', 'ignore')
-        b64h = self.cached_b64h
-        b64p = self.jwt_b64e(json_parser.dumps(payload))
-        jbody = b64h + '.' + b64p
-        mauth = hmac.new(key=secret, msg=jbody, digestmod=self.digestmod)
-        jsign = self.jwt_b64e(mauth.digest())
-        return jbody + '.' + jsign
-
-    def verify_signature(self, body, signature, secret):
-        mauth = hmac.new(key=secret, msg=body, digestmod=self.digestmod)
-        return hmac.compare_digest(self.jwt_b64e(mauth.digest()), signature)
-
-    def load_token(self, token):
-        if isinstance(token, unicode):
-            token = token.encode('utf-8', 'strict')
-        body, sig = token.rsplit('.', 1)
-        b64h, b64b = body.split('.', 1)
-        if b64h != self.cached_b64h:
-            # header not the same
-            raise HTTP(400, u'Invalid JWT Header')
-        secret = self.secret_key
-        tokend = json_parser.loads(self.jwt_b64d(b64b))
-        if self.salt:
-            if callable(self.salt):
-                secret = "%s$%s" % (secret, self.salt(tokend))
-            else:
-                secret = "%s$%s" % (secret, self.salt)
-            if isinstance(secret, unicode):
-                secret = secret.encode('ascii', 'ignore')
-        if not self.verify_signature(body, sig, secret):
-            # signature verification failed
-            raise HTTP(400, u'Token signature is invalid')
-        if self.verify_expiration:
-            now = time.mktime(datetime.datetime.utcnow().timetuple())
-            if tokend['exp'] + self.leeway < now:
-                raise HTTP(400, u'Token is expired')
-        if callable(self.before_authorization):
-            self.before_authorization(tokend)
-        return tokend
-
-    def serialize_auth_session(self, session_auth):
-        """
-        As bad as it sounds, as long as this is rarely used (vs using the token)
-        this is the faster method, even if we ditch session in jwt_token_manager().
-        We (mis)use the heavy default auth mechanism to avoid any further computation,
-        while sticking to a somewhat-stable Auth API.
-        """
-        now = time.mktime(datetime.datetime.utcnow().timetuple())
-        expires = now + self.expiration
-        payload = dict(
-            hmac_key=session_auth['hmac_key'],
-            user_groups=session_auth['user_groups'],
-            user=session_auth['user'].as_dict(),
-            iat=now,
-            exp=expires
-        )
-        return payload
-
-    def refresh_token(self, orig_payload):
-        now = time.mktime(datetime.datetime.utcnow().timetuple())
-        if self.verify_expiration:
-            orig_exp = orig_payload['exp']
-            if orig_exp + self.leeway < now:
-                # token already expired, can't be used for refresh
-                raise HTTP(400, u'Token already expired')
-        orig_iat = orig_payload.get('orig_iat') or orig_payload['iat']
-        if orig_iat + self.refresh_expiration_delta < now:
-            # refreshed too long ago
-            raise HTTP(400, u'Token issued too long ago')
-        expires = now + self.refresh_expiration_delta
-        orig_payload.update(
-            orig_iat=orig_iat,
-            iat=now,
-            exp=expires,
-            hmac_key=web2py_uuid()
-        )
-        self.alter_payload(orig_payload)
-        return orig_payload
-
-    def alter_payload(self, payload):
-        if self.additional_payload:
-            if callable(self.additional_payload):
-                payload = self.additional_payload(payload)
-            elif isinstance(self.additional_payload, dict):
-                payload.update(self.additional_payload)
-        return payload
-
-    def jwt_token_manager(self):
-        """
-        The part that issues (and refreshes) tokens.
-        Used in a controller, given myjwt is the istantiated class, as
-
-            def api_auth():
-                return myjwt.jwt_token_manager()
-
-        Then, a call to /app/c/api_auth/auth with username and password
-        returns a token, while /app/c/api_auth/refresh with the current token
-        issues another token
-        """
-        request = current.request
-        # forget and unlock response
-        if request.args(0) == 'auth':
-            current.session.forget(current.response)
-            username = request.vars[self.user_param]
-            password = request.vars[self.pass_param]
-            valid_user = self.auth.login_bare(username, password)
-            if valid_user:
-                payload = self.serialize_auth_session(current.session.auth)
-                self.alter_payload(payload)
-                return self.generate_token(payload)
-            else:
-                raise HTTP(
-                    401, u'Not Authorized',
-                    **{'WWW-Authenticate': u'JWT realm="%s"' % self.realm})
-        elif request.args(0) == 'refresh':
-            if not self.allow_refresh:
-                raise HTTP(403, u'Refreshing token is not allowed')
-            token = request.vars.token
-            tokend = self.load_token(token)
-            # verification can fail here
-            refreshed = self.refresh_token(tokend)
-            return self.generate_token(refreshed)
-
-    def inject_token(self, tokend):
-        """
-        The real deal, not touching the db but still logging-in the user
-        """
-        self.auth.user = Storage(tokend['user'])
-        self.auth.user_groups = tokend['user_groups']
-        self.auth.hmac_key = tokend['hmac_key']
-
-    def requires_jwt(self, otherwise=None):
-        """
-        The validator that checks for the header or the
-        _token var
-        """
-        request = current.request
-        token_in_header = request.env.http_authorization
-        if token_in_header:
-            parts = token_in_header.split()
-            if parts[0].lower() != self.header_prefix.lower():
-                raise HTTP(400, u'Invalid JWT header')
-            elif len(parts) == 1:
-                raise HTTP(400, u'Invalid JWT header, missing token')
-            elif len(parts) > 2:
-                raise HTTP(400, 'Invalid JWT header, token contains spaces')
-            token = parts[1]
-        else:
-            token = request.vars._token
-        if token and len(token) < self.max_header_length:
-            tokend = self.load_token(token)
-            self.inject_token(tokend)
-        return self.auth.requires(True, otherwise=otherwise)

gluon/form.py

@@ -0,0 +1,407 @@
+import cgi
+import copy_reg
+from gluon import current, URL, DAL
+from gluon.storage import Storage
+from gluon.utils import web2py_uuid
+from gluon.sanitizer import sanitize
+
+# ################################################################
+# New HTML Helpers
+# ################################################################
+
+def xmlescape(text):
+    return cgi.escape(text, True).replace("'", "'")
+
+class TAG(object):    
+
+    def __init__(self, name, *children, **attributes):
+        self.name = name
+        self.children = list(children)
+        self.attributes = attributes
+        for child in self.children:
+            if isinstance(child, TAG):
+                child.parent = self
+
+    def xml(self):
+        name = self.name
+        a = ' '.join('%s="%s"' % 
+                     (k[1:], k[1:] if v is True else xmlescape(unicode(v)))
+                     for k,v in self.attributes.iteritems() 
+                     if k.startswith('_') and not v in (False,None))
+        if a:
+            a = ' '+a
+        if name.endswith('/'):
+            return '<%s%s/>' % (name, a)
+        else:
+            b = ''.join(s.xml() if isinstance(s,TAG) else xmlescape(unicode(s))
+                        for s in self.children)
+            return '<%s%s>%s</%s>' %(name, a, b, name)
+    
+    def __unicode__(self):
+        return self.xml()
+
+    def __str__(self):
+        return self.xml().encode('utf8')
+
+    def __getitem__(self, key):
+        if isinstance(key, int):
+            return self.children[key]
+        else:
+            return self.attributes[key]
+
+    def __setitem__(self, key, value):
+        if isinstance(key, int):
+            self.children[key] = value
+        else:
+            self.attributes[key] =  value
+
+    def append(self, value):
+        self.children.append(value)
+
+    def __delitem__(self,key):
+        if isinstance(key, int):
+            self.children = self.children[:key]+self.children[key+1:]
+        else:
+            del self.attributes[key]
+
+    def __len__(self):
+        return len(self.children)
+
+    def find(self, query):
+        raise NotImplementedError
+
+class METATAG(object):
+
+    def __getattr__(self, name):
+        return self(name)
+
+    def __call__(self, name):
+        return lambda *children, **attributes: TAG(name, *children, **attributes)
+
+tag = METATAG()
+DIV = tag('div')
+SPAN = tag('span')
+LI = tag('li')
+OL = tag('ol')
+UL = tag('ul')
+A  = tag('a')
+H1 = tag('h1')
+H2 = tag('h2')
+H3 = tag('h3')
+H4 = tag('h4')
+H5 = tag('h5')
+H6 = tag('h6')
+EM = tag('em')
+TR = tag('tr')
+TD = tag('td')
+TH = tag('th')
+IMG = tag('img/')
+FORM = tag('form')
+HEAD = tag('head')
+BODY = tag('body')
+TABLE = tag('table')
+INPUT = tag('input/')
+LABEL = tag('label')
+STRONG = tag('strong')
+SELECT = tag('select')
+OPTION = tag('option')
+TEXTAREA = tag('textarea')
+
+# ################################################################
+# New XML Helpers
+# ################################################################
+
+class XML(TAG):
+    """
+    use it to wrap a string that contains XML/HTML so that it will not be
+    escaped by the template
+
+    Examples:
+
+    >>> XML('<h1>Hello</h1>').xml()
+    '<h1>Hello</h1>'
+    """
+
+    def __init__(
+        self,
+        text,
+        sanitize=False,
+        permitted_tags=[
+            'a','b','blockquote','br/','i','li','ol','ul','p','cite',
+            'code','pre','img/','h1', 'h2', 'h3', 'h4', 'h5', 'h6',
+            'table', 'tr', 'td', 'div','strong', 'span'],
+        allowed_attributes={
+            'a': ['href', 'title', 'target'],
+            'img': ['src', 'alt'],
+            'blockquote': ['type'],
+            'td': ['colspan']},
+        ):
+        """
+        Args:
+            text: the XML text
+            sanitize: sanitize text using the permitted tags and allowed
+                attributes (default False)
+            permitted_tags: list of permitted tags (default: simple list of
+                tags)
+            allowed_attributes: dictionary of allowed attributed (default
+                for A, IMG and BlockQuote).
+                The key is the tag; the value is a list of allowed attributes.
+        """
+
+        if sanitize:
+            text = sanitize(text, permitted_tags, allowed_attributes)
+        if isinstance(text, unicode):
+            text = text.encode('utf8', 'xmlcharrefreplace')
+        elif not isinstance(text, str):
+            text = str(text)
+        self.text = text
+
+    def xml(self):
+        return self.text
+
+    def __str__(self):
+        return self.text
+
+    def __add__(self, other):
+        return '%s%s' % (self, other)
+
+    def __radd__(self, other):
+        return '%s%s' % (other, self)
+
+    def __cmp__(self, other):
+        return cmp(str(self), str(other))
+
+    def __hash__(self):
+        return hash(str(self))
+
+    def __getitem__(self, i):
+        return str(self)[i]
+
+    def __getslice__(self, i, j):
+        return str(self)[i:j]
+
+    def __iter__(self):
+        for c in str(self):
+            yield c
+
+    def __len__(self):
+        return len(str(self))
+
+def XML_unpickle(data):
+    return XML(marshal.loads(data))
+
+def XML_pickle(data):
+    return XML_unpickle, (marshal.dumps(str(data)),)
+copy_reg.pickle(XML, XML_pickle, XML_unpickle)
+
+# ################################################################
+# Simple Form Style Function (example for more complex styles)
+# ################################################################
+
+def FormStyleDefault(table, vars, errors, readonly, deletable):
+    
+    form = FORM(TABLE(),_method='POST',_action='#',_enctype='multipart/form-data')
+    for field in table:
+        
+        input_id = '%s_%s' % (field.tablename, field.name)        
+        value = field.formatter(vars.get(field.name))
+        error = errors.get(field.name)
+        field_class = field.type.split()[0].replace(':','-')
+
+        if field.type == 'blob': # never display blobs (mistake?)
+            continue
+        elif readonly or field.type=='id':
+            if not field.readable:
+                continue
+            else:
+                control = field.represent and field.represent(value) or value or ''
+        elif not field.writable:
+            continue
+        elif field.widget:
+            control = field.widget(table, value)
+        elif field.type == 'text':
+            control = TEXTAREA(value or '', _id=input_id,_name=field.name)
+        elif field.type == 'boolean':
+            control = INPUT(_type='checkbox', _id=input_id, _name=field.name,
+                            _value='ON', _checked = value)
+        elif field.type == 'upload':
+            control = DIV(INPUT(_type='file', _id=input_id, _name=field.name))
+            if value:
+                control.append(A('download',
+                                 _href=URL('default','download',args=value)))
+                control.append(INPUT(_type='checkbox',_value='ON',
+                                     _name='_delete_'+field.name))
+                control.append('(check to remove)')
+        elif hasattr(field.requires, 'options'):
+            multiple = field.type.startswith('list:')
+            value = value if isinstance(value, list) else [value]
+            options = [OPTION(v,_value=k,_selected=(k in value)) 
+                       for k,v in field.requires.options()]
+            control = SELECT(*options, _id=input_id, _name=field.name,
+                              _multiple=multiple)
+        else:
+            field_type = 'password' if field.type == 'password' else 'text'
+            control = INPUT(_type=field_type, _id=input_id, _name=field.name,
+                            _value=value, _class=field_class)
+            
+        form[0].append(TR(TD(LABEL(field.label,_for=input_id)),
+                          TD(control,DIV(error,_class='error') if error else ''),
+                          TD(field.comment or '')))
+        
+    td = TD(INPUT(_type='submit',_value='Submit'))
+    if deletable:
+        td.append(INPUT(_type='checkbox',_value='ON',_name='_delete'))
+        td.append('(check to delete)')
+    form[0].append(TR(TD(),td,TD()))                         
+    return form
+
+# ################################################################
+# Form object (replaced SQLFORM)
+# ################################################################
+
+class Form(object):
+    """
+    Usage in web2py controller:
+
+       def index():
+           form = Form(db.thing, record=1)
+           if form.accepted: ...
+           elif form.errors: ...
+           else: ...
+           return dict(form=form)
+           
+    Arguments:
+    - table: a DAL table or a list of fields (equivalent to old SQLFORM.factory)
+    - record: a DAL record or record id
+    - readonly: set to True to make a readonly form
+    - deletable: set to False to disallow deletion of record
+    - formstyle: a function that renders the form using helpers (FormStyleDefault)
+    - dbio: set to False to prevent any DB write
+    - keepvalues: (NOT IMPLEMENTED)
+    - formname: the optional name of this form
+    - csrf: set to False to disable CRSF protection
+    """
+    
+    def __init__(self,
+                 table, 
+                 record=None, 
+                 readonly=False, 
+                 deletable=True,
+                 formstyle=FormStyleDefault, 
+                 dbio=True,
+                 keepvalues=False,
+                 formname=False,
+                 csrf=True):
+
+        if isinstance(table, list):
+            dbio = False
+            # mimic a table from a list of fields without calling define_table
+            formname = formname or 'none'
+            for field in table: field.tablename = formname
+ 
+        if isinstance(record, (int, long, basestring)):
+            record_id = int(str(record))
+            self.record = table[record_id]
+        else:
+            self.record = record
+
+        self.table = table
+        self.readonly = readonly
+        self.deletable = deletable and not readonly and self.record
+        self.formstyle = formstyle
+        self.dbio = dbio
+        self.keepvalues = True if keepvalues or self.record else False
+        self.csrf = csrf
+        self.vars = Storage()
+        self.errors = Storage()
+        self.submitted = False
+        self.deleted = False
+        self.accepted = False
+        self.cached_helper = False
+        self.formname = formname or table._tablename
+        self.formkey = None
+
+        request = current.request
+        session = current.session
+        post_vars = request.post_vars
+
+        if readonly or request.env.request_method=='GET':
+            if self.record:
+                self.vars = self.record
+        else:
+            print post_vars
+            self.submitted = True            
+            # check for CSRF
+            if csrf and self.formname in (session._formkeys or {}):
+                self.formkey = session._formkeys[self.formname]                
+            # validate fields
+            if not csrf or post_vars._formkey == self.formkey:
+                if not post_vars._delete:
+                    for field in self.table:
+                        if field.writable:                            
+                            value = post_vars.get(field.name)
+                            (value, error) = field.validate(value)
+                            if field.type == 'upload':
+                                delete = post_vars.get('_delete_'+field.name)
+                                if value is not None and hasattr(value,'file'):
+                                    value = field.store(value.file, 
+                                                        value.filename,
+                                                        field.uploadfolder)
+                                elif self.record and not delete:
+                                    value = self.record.get(field.name)
+                                else:
+                                    value = None
+                            self.vars[field.name] = value
+                            if error:
+                                self.errors[field.name] = error
+                    if self.record:
+                        self.vars.id = self.record.id
+                    if not self.errors:
+                        self.accepted = True
+                        if dbio:
+                            if self.record:
+                                self.record.update_record(**self.vars)
+                            else:
+                                # warning, should we really insert if record
+                                self.vars.id = self.table.insert(**self.vars)
+                elif dbio:
+                    self.deleted = True
+                    self.record.delete_record()
+        # store key for future CSRF
+        if csrf:
+            if not session._formkeys:
+                session._formkeys = {}
+            if self.formname not in session._formkeys:
+                session._formkeys[self.formname] = web2py_uuid()
+            self.formkey = session._formkeys[self.formname]
+
+    def clear():
+        self.vars.clear()
+        self.errors.clear()
+        for field in self.table:
+            self.vars[field.name] = field.default
+
+    def helper(self):
+        if not self.cached_helper:
+            cached_helper = self.formstyle(self.table,
+                                           self.vars,
+                                           self.errors,
+                                           self.readonly,
+                                           self.deletable)
+            if self.csrf:
+                cached_helper.append(INPUT(_type='hidden',_name='_formkey',
+                                           _value=self.formkey))
+            self.cached_helper = cached_helper
+        return cached_helper
+
+    def xml(self):
+        return self.helper().xml()
+
+    def __unicode__(self):
+        return self.xml()
+
+    def __str__(self):
+        return self.xml().encode('utf8')
+
+if __name__=='__main__':
+    print(DIV(SPAN('this',STRONG('a test'),XML('1<2')),_id=1,_class="my class"))

gluon/html.py

@@ -1859,6 +1859,8 @@ class INPUT(DIV):
                 try:
                     (value, errors) = validator(value)
                 except:
+                    import traceback
+                    print traceback.format_exc()
                     msg = "Validation error, field:%s %s" % (name,validator)
                     raise Exception(msg)
                 if not errors is None:
@@ -2646,7 +2648,7 @@ def test():
     >>> form=FORM(INPUT(value="Hello World", _name="var", requires=IS_MATCH('^\w+$')))
     >>> isinstance(form.as_dict(), dict)
     True
-    >>> form.as_dict(flat=True).has_key("vars")
+    >>> "vars" in form.as_dict(flat=True)
     True
     >>> isinstance(form.as_json(), basestring) and len(form.as_json(sanitize=False)) > 0
     True

gluon/sqlhtml.py

@@ -1504,13 +1504,12 @@ class SQLFORM(FORM):
             hideerror=hideerror,
             **kwargs
         )
-
-        self.deleted = \
-            request_vars.get(self.FIELDNAME_REQUEST_DELETE, False)
+        
+        self.deleted = request_vars.get(self.FIELDNAME_REQUEST_DELETE, False)
 
         self.custom.end = CAT(self.hidden_fields(), self.custom.end)
 
-        auch = self.record_id and self.errors and self.deleted
+        delete_exception = self.record_id and self.errors and self.deleted
 
         if self.record_changed and self.detect_record_change:
             message_onchange = \
@@ -1522,8 +1521,9 @@ class SQLFORM(FORM):
             if message_onchange is not None:
                 current.response.flash = message_onchange
             return ret
-        elif (not ret) and (not auch):
-            # auch is true when user tries to delete a record
+
+        elif (not ret) and (not delete_exception):
+            # delete_exception is true when user tries to delete a record
             # that does not pass validation, yet it should be deleted
             for fieldname in self.fields:
 
@@ -1679,9 +1679,6 @@ class SQLFORM(FORM):
             elif field.type == 'double':
                 if value is not None:
                     fields[fieldname] = safe_float(value)
-            elif field.type in ('string', 'text'):
-                if fieldname in self.request_vars:
-                    fields[fieldname] = self.request_vars[fieldname]
 
         for fieldname in self.vars:
             if fieldname != 'id' and fieldname in self.table.fields\
@@ -1723,6 +1720,7 @@ class SQLFORM(FORM):
                                        self.id_field_name]).update(**fields)
                 else:
                     self.vars.id = self.table.insert(**fields)
+
         self.accepted = ret
         return ret
 

gluon/tools.py

@@ -32,8 +32,12 @@ import cStringIO
 import ConfigParser
 import email.utils
 import random
+import hmac
+import hashlib
+import json
 from email import MIMEBase, MIMEMultipart, MIMEText, Encoders, Header, message_from_string, Charset
 
+from gluon.serializers import json_parser
 from gluon.contenttype import contenttype
 from gluon.storage import Storage, StorageList, Settings, Messages
 from gluon.utils import web2py_uuid
@@ -1136,6 +1140,304 @@ def addrow(form, a, b, c, style, _id, position=-1):
                                     TD(c, _class='w2p_fc'), _id=_id))
 
 
+class AuthJWT(object):
+
+    """
+    If left externally, this needs the usual "singleton" approach.
+    Given I (we) don't know if to include in auth yet, let's stick to basics.
+
+    Args:
+     - secret_key: the secret. Without salting, an attacker knowing this can impersonate
+                   any user
+     - algorithm : uses as they are in the JWT specs, HS256, HS384 or HS512 basically means
+                   signing with HMAC with a 256, 284 or 512bit hash
+     - verify_expiration : verifies the expiration checking the exp claim
+     - leeway: allow n seconds of skew when checking for token expiration
+     - expiration : how many seconds a token may be valid
+     - allow_refresh: enable the machinery to get a refreshed token passing a not-already-expired
+                      token
+     - refresh_expiration_delta: to avoid continous refresh of the token
+     - header_prefix : self-explanatory. "JWT" and "Bearer" seems to be the emerging standards
+     - jwt_add_header: a dict holding additional mappings to the header. by default only alg and typ are filled
+     - user_param: the name of the parameter holding the username when requesting a token. Can be useful, e.g, for
+                   email-based authentication, with "email" as a parameter
+     - pass_param: same as above, but for the password
+     - realm: self-explanatory
+     - salt: can be static or a function that takes the payload as an argument.
+             Example:
+             def mysalt(payload):
+                return payload['hmac_key'].split('-')[0]
+     - additional_payload: can be a dict to merge with the payload or a function that takes
+                           the payload as input and returns the modified payload
+                           Example:
+                           def myadditional_payload(payload):
+                               payload['my_name_is'] = 'bond,james bond'
+                               return payload
+     - before_authorization: can be a callable that takes the deserialized token (a dict) as input.
+                             Gets called right after signature verification but before the actual
+                             authorization takes place. You can raise with HTTP a proper error message
+                             Example:
+                             def mybefore_authorization(tokend):
+                                 if not tokend['my_name_is'] == 'bond,james bond':
+                                     raise HTTP(400, u'Invalid JWT my_name_is claim')
+     - max_header_length: check max length to avoid load()ing unusually large tokens (could mean crafted, e.g. in a DDoS.)
+
+    Basic Usage:
+    in models (or the controller needing it)
+
+        myjwt = AuthJWT(auth, secret_key='secret')
+
+    in the controller issuing tokens
+
+        def login_and_take_token():
+            return myjwt.jwt_token_manager()
+
+    A call then to /app/controller/login_and_take_token/auth with username and password returns the token
+    A call to /app/controller/login_and_take_token/refresh with the original token returns the refreshed token
+
+    To protect a function with JWT
+
+        @myjwt.allows_jwt()
+        @auth.requires_login()
+        def protected():
+            return '%s$%s' % (request.now, auth.user_id)
+
+    """
+
+    def __init__(self, 
+                 auth,
+                 secret_key,
+                 algorithm='HS256',
+                 verify_expiration=True,
+                 leeway=30,
+                 expiration=60 * 5,
+                 allow_refresh=True,
+                 refresh_expiration_delta=60 * 60,
+                 header_prefix='Bearer',
+                 jwt_add_header=None,
+                 user_param='username',
+                 pass_param='password',
+                 realm='Login required',
+                 salt=None,
+                 additional_payload=None,
+                 before_authorization=None,
+                 max_header_length=4*1024,
+                 ):
+        self.secret_key = secret_key
+        self.auth = auth
+        self.algorithm = algorithm
+        if self.algorithm not in ('HS256', 'HS384', 'HS512'):
+            raise NotImplementedError('Algoritm %s not allowed' % algorithm)
+        self.verify_expiration = verify_expiration
+        self.leeway = leeway
+        self.expiration = expiration
+        self.allow_refresh = allow_refresh
+        self.refresh_expiration_delta = refresh_expiration_delta
+        self.header_prefix = header_prefix
+        self.jwt_add_header = jwt_add_header or {}
+        base_header = {'alg': self.algorithm, 'typ': 'JWT'}
+        for k, v in self.jwt_add_header.iteritems():
+            base_header[k] = v
+        self.cached_b64h = self.jwt_b64e(json_parser.dumps(base_header))
+        digestmod_mapping = {
+            'HS256': hashlib.sha256,
+            'HS384': hashlib.sha384,
+            'HS512': hashlib.sha512
+        }
+        self.digestmod = digestmod_mapping[algorithm]
+        self.user_param = user_param
+        self.pass_param = pass_param
+        self.realm = realm
+        self.salt = salt
+        self.additional_payload = additional_payload
+        self.before_authorization = before_authorization
+        self.max_header_length = max_header_length
+
+    @staticmethod
+    def jwt_b64e(string):
+        if isinstance(string, unicode):
+            string = string.encode('uft-8', 'strict')
+        return base64.urlsafe_b64encode(string).strip(b'=')
+
+    @staticmethod
+    def jwt_b64d(string):
+        """base64 decodes a single bytestring (and is tolerant to getting
+        called with a unicode string).
+        The result is also a bytestring.
+        """
+        if isinstance(string, unicode):
+            string = string.encode('ascii', 'ignore')
+        return base64.urlsafe_b64decode(string + '=' * (-len(string) % 4))
+
+    def generate_token(self, payload):
+        secret = self.secret_key
+        if self.salt:
+            if callable(self.salt):
+                secret = "%s$%s" % (secret, self.salt(payload))
+            else:
+                secret = "%s$%s" % (secret, self.salt)
+            if isinstance(secret, unicode):
+                secret = secret.encode('ascii', 'ignore')
+        b64h = self.cached_b64h
+        b64p = self.jwt_b64e(json_parser.dumps(payload))
+        jbody = b64h + '.' + b64p
+        mauth = hmac.new(key=secret, msg=jbody, digestmod=self.digestmod)
+        jsign = self.jwt_b64e(mauth.digest())
+        return jbody + '.' + jsign
+
+    def verify_signature(self, body, signature, secret):
+        mauth = hmac.new(key=secret, msg=body, digestmod=self.digestmod)
+        return hmac.compare_digest(self.jwt_b64e(mauth.digest()), signature)
+
+    def load_token(self, token):
+        if isinstance(token, unicode):
+            token = token.encode('utf-8', 'strict')
+        body, sig = token.rsplit('.', 1)
+        b64h, b64b = body.split('.', 1)
+        if b64h != self.cached_b64h:
+            # header not the same
+            raise HTTP(400, u'Invalid JWT Header')
+        secret = self.secret_key
+        tokend = json_parser.loads(self.jwt_b64d(b64b))
+        if self.salt:
+            if callable(self.salt):
+                secret = "%s$%s" % (secret, self.salt(tokend))
+            else:
+                secret = "%s$%s" % (secret, self.salt)
+            if isinstance(secret, unicode):
+                secret = secret.encode('ascii', 'ignore')
+        if not self.verify_signature(body, sig, secret):
+            # signature verification failed
+            raise HTTP(400, u'Token signature is invalid')
+        if self.verify_expiration:
+            now = time.mktime(datetime.datetime.utcnow().timetuple())
+            if tokend['exp'] + self.leeway < now:
+                raise HTTP(400, u'Token is expired')
+        if callable(self.before_authorization):
+            self.before_authorization(tokend)
+        return tokend
+
+    def serialize_auth_session(self, session_auth):
+        """
+        As bad as it sounds, as long as this is rarely used (vs using the token)
+        this is the faster method, even if we ditch session in jwt_token_manager().
+        We (mis)use the heavy default auth mechanism to avoid any further computation,
+        while sticking to a somewhat-stable Auth API.
+        """
+        now = time.mktime(datetime.datetime.utcnow().timetuple())
+        expires = now + self.expiration
+        payload = dict(
+            hmac_key=session_auth['hmac_key'],
+            user_groups=session_auth['user_groups'],
+            user=session_auth['user'].as_dict(),
+            iat=now,
+            exp=expires
+        )
+        return payload
+
+    def refresh_token(self, orig_payload):
+        now = time.mktime(datetime.datetime.utcnow().timetuple())
+        if self.verify_expiration:
+            orig_exp = orig_payload['exp']
+            if orig_exp + self.leeway < now:
+                # token already expired, can't be used for refresh
+                raise HTTP(400, u'Token already expired')
+        orig_iat = orig_payload.get('orig_iat') or orig_payload['iat']
+        if orig_iat + self.refresh_expiration_delta < now:
+            # refreshed too long ago
+            raise HTTP(400, u'Token issued too long ago')
+        expires = now + self.refresh_expiration_delta
+        orig_payload.update(
+            orig_iat=orig_iat,
+            iat=now,
+            exp=expires,
+            hmac_key=web2py_uuid()
+        )
+        self.alter_payload(orig_payload)
+        return orig_payload
+
+    def alter_payload(self, payload):
+        if self.additional_payload:
+            if callable(self.additional_payload):
+                payload = self.additional_payload(payload)
+            elif isinstance(self.additional_payload, dict):
+                payload.update(self.additional_payload)
+        return payload
+
+    def jwt_token_manager(self):
+        """
+        The part that issues (and refreshes) tokens.
+        Used in a controller, given myjwt is the istantiated class, as
+
+            def api_auth():
+                return myjwt.jwt_token_manager()
+
+        Then, a call to /app/c/api_auth/auth with username and password
+        returns a token, while /app/c/api_auth/refresh with the current token
+        issues another token
+        """
+        request = current.request
+        response = current.response
+        session = current.session
+        # forget and unlock response
+        if request.vars.token:
+            if not self.allow_refresh:
+                raise HTTP(403, u'Refreshing token is not allowed')
+            token = request.vars.token
+            tokend = self.load_token(token)
+            # verification can fail here
+            refreshed = self.refresh_token(tokend)
+            ret = {'token':self.generate_token(refreshed)}
+        elif self.user_param in request.vars and self.pass_param in request.vars:
+            session.forget(response)
+            username = request.vars[self.user_param]
+            password = request.vars[self.pass_param]
+            valid_user = self.auth.login_bare(username, password)
+            if valid_user:
+                payload = self.serialize_auth_session(current.session.auth)
+                self.alter_payload(payload)
+                ret = {'token':self.generate_token(payload)}
+            else:
+                raise HTTP(
+                    401, u'Not Authorized', 
+                    **{'WWW-Authenticate': u'JWT realm="%s"' % self.realm})
+        else:
+            raise HTTP(400, u'Must pass token for refresh or username and password for login')
+        response.headers['content-type'] = 'application/json'
+        return json.dumps(ret)
+
+    def inject_token(self, tokend):
+        """
+        The real deal, not touching the db but still logging-in the user
+        """
+        self.auth.user = Storage(tokend['user'])
+        self.auth.user_groups = tokend['user_groups']
+        self.auth.hmac_key = tokend['hmac_key']
+
+    def allows_jwt(self, otherwise=None):
+        """
+        The validator that checks for the header or the
+        _token var
+        """
+        request = current.request
+        token_in_header = request.env.http_authorization
+        if token_in_header:
+            parts = token_in_header.split()
+            if parts[0].lower() != self.header_prefix.lower():
+                raise HTTP(400, u'Invalid JWT header')
+            elif len(parts) == 1:
+                raise HTTP(400, u'Invalid JWT header, missing token')
+            elif len(parts) > 2:
+                raise HTTP(400, 'Invalid JWT header, token contains spaces')
+            token = parts[1]
+        else:
+            token = request.vars._token
+        if token and len(token) < self.max_header_length:
+            tokend = self.load_token(token)
+            self.inject_token(tokend)
+        return self.auth.requires(True, otherwise=otherwise)
+
+
 class Auth(object):
 
     default_settings = dict(
@@ -1424,7 +1726,7 @@ class Auth(object):
                  hmac_key=None, controller='default', function='user',
                  cas_provider=None, signature=True, secure=False,
                  csrf_prevention=True, propagate_extension=None,
-                 url_index=None):
+                 url_index=None, jwt=None):
 
         ## next two lines for backward compatibility
         if not db and environment and isinstance(environment, DAL):
@@ -1544,6 +1846,8 @@ class Auth(object):
             self.define_signature()
         else:
             self.signature = None
+        
+        self.jwt_handler = jwt and AuthJWT(self, **jwt)
 
     def get_vars_next(self):
         next = current.request.vars._next
@@ -1612,7 +1916,7 @@ class Auth(object):
                        'reset_password', 'request_reset_password',
                        'change_password', 'profile', 'groups',
                        'impersonate', 'not_authorized', 'confirm_registration',
-                       'bulk_register','manage_tokens'):
+                       'bulk_register','manage_tokens','jwt'):
             if len(request.args) >= 2 and args[0] == 'impersonate':
                 return getattr(self, args[0])(request.args[1])
             else:
@@ -1627,7 +1931,7 @@ class Auth(object):
             elif args(1) == self.settings.cas_actions['proxyvalidate']:
                 return self.cas_validate(version=2, proxy=True)
             elif args(1) == self.settings.cas_actions['logout']:
-                return self.logout(next=request.vars.service or DEFAULT)
+                return self.logout(next=request.vars.service or DEFAULT)        
         else:
             raise HTTP(404)
 
@@ -2652,8 +2956,8 @@ class Auth(object):
                         user = table_user(**{username: entered_username})
                     if user:
                         # user in db, check if registration pending or disabled
-                        temp_user = user
-                        if temp_user.registration_key == 'pending':
+                        temp_user = user                        
+                        if (temp_user.registration_key or '').startswith('pending'):
                             response.flash = self.messages.registration_pending
                             return form
                         elif temp_user.registration_key in ('disabled', 'blocked'):
@@ -3028,7 +3332,11 @@ class Auth(object):
                    DIV(_id="pre-reg", *self.settings.pre_registration_div),
                    '', formstyle, '')
 
-        table_user.registration_key.default = key = web2py_uuid()
+        key = web2py_uuid()
+        if self.settings.registration_requires_approval:
+            key = 'pending-'+key
+
+        table_user.registration_key.default = key
         if form.accepts(request, session if self.csrf_prevention else None,
                         formname='register',
                         onvalidation=onvalidation,
@@ -3242,11 +3550,12 @@ class Auth(object):
                         formname='retrieve_password', dbio=False,
                         onvalidation=onvalidation, hideerror=self.settings.hideerror):
             user = table_user(email=form.vars.email)
+            key = user.registration_key
             if not user:
                 current.session.flash = \
                     self.messages.invalid_email
                 redirect(self.url(args=request.args))
-            elif user.registration_key in ('pending', 'disabled', 'blocked'):
+            elif key in ('pending', 'disabled', 'blocked') or (key or '').startswith('pending'):
                 current.session.flash = \
                     self.messages.registration_pending
                 redirect(self.url(args=request.args))
@@ -3450,6 +3759,11 @@ class Auth(object):
         except Exception:
             session.flash = self.messages.invalid_reset_password
             redirect(next, client_side=self.settings.client_side)
+        
+        key = user.registration_key
+        if key in ('pending', 'disabled', 'blocked') or (key or '').startswith('pending'):
+            session.flash = self.messages.registration_pending
+            redirect(next, client_side=self.settings.client_side)
 
         if onvalidation is DEFAULT:
             onvalidation = self.settings.reset_password_onvalidation
@@ -3544,11 +3858,12 @@ class Auth(object):
                         onvalidation=onvalidation,
                         hideerror=self.settings.hideerror):
             user = table_user(**{userfield:form.vars.get(userfield)})
+            key = user.registration_key 
             if not user:
                 session.flash = self.messages['invalid_%s' % userfield]
                 redirect(self.url(args=request.args),
                          client_side=self.settings.client_side)
-            elif user.registration_key in ('pending', 'disabled', 'blocked'):
+            elif key in ('pending', 'disabled', 'blocked') or (key or '').startswith('pending'):
                 session.flash = self.messages.registration_pending
                 redirect(self.url(args=request.args),
                          client_side=self.settings.client_side)
@@ -3727,6 +4042,50 @@ class Auth(object):
             for callback in onaccept:
                 callback(form)
 
+    def jwt(self):
+        """
+        To use JWT authentication:
+        1) instantiate auth with
+
+            auth = Auth(db, jwt = {'secret_key':'secret'})
+
+        where 'secret' is your own secret string. 
+
+        2) Secorate functions that require login but should accept the JWT token credentials:
+
+            @auth.allows_jwt()
+            @auth.requires_login()
+            def myapi(): return 'hello %s' % auth.user.email
+    
+        Notice jwt is allowed but not required. if user is logged in, myapi is accessible.
+
+        3) Use it!
+
+        Now API users can obtain a token with
+
+            http://.../app/default/user/jwt?username=...&password=....
+
+        (returns json object with a token attribute)
+        API users can refresh an existing token with
+
+            http://.../app/default/user/jwt?token=...
+
+        they can authenticate themselves when calling http:/.../myapi by injecting a header
+
+            Authorization: Bearer <the jwt token>
+
+        Any additional attributes in the jwt argument of Auth() below:
+
+           auth = Auth(db, jwt = {...})
+
+        are passed to the constructor of class AuthJWT. Look there for documentation.
+        """
+        if not self.jwt_handler:
+            raise HTTP(400, "Not authorized")
+        else:
+            current.response.headers['content-type'] = 'application/json'
+            raise HTTP(200, self.jwt_handler.jwt_token_manager())
+        
     def is_impersonating(self):
         return self.is_logged_in() and 'impersonator' in current.session.auth
 
@@ -3825,6 +4184,12 @@ class Auth(object):
             raise HTTP(403, 'ACCESS DENIED')
         return self.messages.access_denied
 
+    def allows_jwt(self, otherwise=None):
+        if not self.jwt_handler:
+            raise HTTP(400, "Not authorized")
+        else:
+            return self.jwt_handler.allows_jwt()
+
     def requires(self, condition, requires_login=True, otherwise=None):
         """
         Decorator that prevents access to action if not logged in
@@ -4328,7 +4693,7 @@ class Auth(object):
         if resolve:
             if slug:
                 wiki = self._wiki.read(slug, force_render)
-                if isinstance(wiki, dict) and wiki.has_key('content'):  # FIXME: .has_key() is deprecated
+                if isinstance(wiki, dict) and 'content' in wiki:
                     # We don't want to return a dict object, just the wiki
                     wiki = wiki['content']
             else:
@@ -5345,7 +5710,7 @@ class Service(object):
 
         def return_error(id, code, message=None, data=None):
             error = {'code': code}
-            if Service.jsonrpc_errors.has_key(code):
+            if code in Service.jsonrpc_errors:
                 error['message'] = Service.jsonrpc_errors[code][0]
                 error['data'] = Service.jsonrpc_errors[code][1]
             if message is not None:

gluon/validators.py

@@ -22,7 +22,7 @@ import decimal
 import unicodedata
 from cStringIO import StringIO
 from gluon.utils import simple_hash, web2py_uuid, DIGEST_ALG_BY_SIZE
-from pydal.objects import FieldVirtual, FieldMethod
+from pydal.objects import Field, FieldVirtual, FieldMethod
 
 regex_isint = re.compile('^[+-]?\d+$')
 
@@ -509,34 +509,44 @@ class IS_IN_DB(Validator):
         zero='',
         sort=False,
         _and=None,
-        left=None
+        left=None,
+        delimiter=None,
+        auto_add=False,
     ):
         from pydal.objects import Table
-        if isinstance(field, Table):
-            field = field._id
-
         if hasattr(dbset, 'define_table'):
             self.dbset = dbset()
         else:
             self.dbset = dbset
+
+        if isinstance(field, Table):
+            field = field._id
+        elif isinstance(field, str):
+            items = field.split('.')
+            if len(items)==1: items+=['id']
+            field = self.dbset.db[items[0]][items[1]]
+
         (ktable, kfield) = str(field).split('.')
         if not label:
             label = '%%(%s)s' % kfield
         if isinstance(label, str):
             if regex1.match(str(label)):
                 label = '%%(%s)s' % str(label).split('.')[-1]
-            ks = regex2.findall(label)
-            if kfield not in ks:
-                ks += [kfield]
-            fields = ks
+            fieldnames = regex2.findall(label)
+            if kfield not in fieldnames:
+                fieldnames.append(kfield) # kfield must be last
+        elif isinstance(label, Field):
+            fieldnames = [label.name, kfield] # kfield must be last
+            label = '%%(%s)s' % label.name
+        elif callable(label):
+            fieldnames = '*'
         else:
-            ks = [kfield]
-            fields = 'all'
-        self.fields = fields
+            raise NotImplementedError
+        self.field = field # the lookup field
+        self.fieldnames = fieldnames # fields requires to build the formatting
         self.label = label
         self.ktable = ktable
         self.kfield = kfield
-        self.ks = ks
         self.error_message = error_message
         self.theset = None
         self.orderby = orderby
@@ -548,6 +558,8 @@ class IS_IN_DB(Validator):
         self.sort = sort
         self._and = _and
         self.left = left
+        self.delimiter = delimiter
+        self.auto_add = auto_add
 
     def set_self_id(self, id):
         if self._and:
@@ -555,10 +567,10 @@ class IS_IN_DB(Validator):
 
     def build_set(self):
         table = self.dbset.db[self.ktable]
-        if self.fields == 'all':
+        if self.fieldnames == '*':
             fields = [f for f in table]
         else:
-            fields = [table[k] for k in self.fields]
+            fields = [table[k] for k in self.fieldnames]
         ignore = (FieldVirtual, FieldMethod)
         fields = filter(lambda f: not isinstance(f, ignore), fields)
         if self.dbset.db._dbname != 'gae':
@@ -591,18 +603,42 @@ class IS_IN_DB(Validator):
             items.insert(0, ('', self.zero))
         return items
 
+    def maybe_add(self, table, fieldname, value):
+        d = {fieldname: value}
+        record = table(**d)
+        if record:
+            return record.id
+        else:
+            return table.insert(**d)
+
     def __call__(self, value):
         table = self.dbset.db[self.ktable]
         field = table[self.kfield]
+
         if self.multiple:
             if self._and:
                 raise NotImplementedError
             if isinstance(value, list):
                 values = value
+            elif self.delimiter:
+                values = value.split(self.delimiter) # because of autocomplete
             elif value:
                 values = [value]
             else:
                 values = []
+
+            if self.field.type in ('id','integer'):
+                new_values = []
+                for value in values:
+                    if isinstance(value,(int,long)) or value.isdigit():
+                        value = int(value)
+                    elif self.auto_add:
+                        value = self.maybe_add(table, self.fieldnames[0], value)
+                    else:
+                        return (values, translate(self.error_message))
+                    new_values.append(value)
+                values = new_values
+
             if isinstance(self.multiple, (tuple, list)) and \
                     not self.multiple[0] <= len(values) < self.multiple[1]:
                 return (values, translate(self.error_message))
@@ -621,18 +657,32 @@ class IS_IN_DB(Validator):
                         return (values, None)
                 elif count(values) == len(values):
                     return (values, None)
-        elif self.theset:
-            if str(value) in self.theset:
-                if self._and:
-                    return self._and(value)
-                else:
-                    return (value, None)
         else:
-            if self.dbset(field == value).count():
-                if self._and:
-                    return self._and(value)
+            if self.field.type in ('id','integer'):
+                if isinstance(value,(int,long)) or value.isdigit():
+                    value = int(value)
+                elif self.auto_add:
+                    value = self.maybe_add(table, self.fieldnames[0], value)
                 else:
-                    return (value, None)
+                    return (value, translate(self.error_message))
+
+                try:
+                    value = int(value)
+                except TypeError:
+                    return (values, translate(self.error_message))
+
+            if self.theset:
+                if str(value) in self.theset:
+                    if self._and:
+                        return self._and(value)
+                    else:
+                        return (value, None)
+            else:
+                if self.dbset(field == value).count():
+                    if self._and:
+                        return self._and(value)
+                    else:
+                        return (value, None)
         return (value, translate(self.error_message))
 
 

gluon/widget.py

@@ -40,8 +40,8 @@ ProgramInfo = '''%s
                  %s
                  %s''' % (ProgramName, ProgramAuthor, ProgramVersion)
 
-if not sys.version[:3] in ['2.5', '2.6', '2.7']:
-    msg = 'Warning: web2py requires Python 2.5, 2.6 or 2.7 but you are running:\n%s'
+if not sys.version[:3] in ['2.6', '2.7']:
+    msg = 'Warning: web2py requires Python 2.6 or 2.7 but you are running:\n%s'
     msg = msg % sys.version
     sys.stderr.write(msg)
 
@@ -56,8 +56,8 @@ def run_system_tests(options):
     major_version = sys.version_info[0]
     minor_version = sys.version_info[1]
     if major_version == 2:
-        if minor_version in (5, 6):
-            sys.stderr.write("Python 2.5 or 2.6\n")
+        if minor_version in (6,):
+            sys.stderr.write('Python 2.6\n')
             ret = subprocess.call(['unit2', '-v', 'gluon.tests'])
         elif minor_version in (7,):
             call_args = [sys.executable, '-m', 'unittest', '-v', 'gluon.tests']
@@ -1117,12 +1117,12 @@ def start(cron=True):
             if hasattr(options, key):
                 setattr(options, key, getattr(options2, key))
 
-    logfile0 = os.path.join('extras', 'examples', 'logging.example.conf')
+    logfile0 = os.path.join('examples', 'logging.example.conf')
     if not os.path.exists('logging.conf') and os.path.exists(logfile0):
         import shutil
         sys.stdout.write("Copying logging.conf.example to logging.conf ... ")
-        shutil.copyfile('logging.example.conf', logfile0)
-        sys.stdout.write("OK\n")
+        shutil.copyfile(logfile0, 'logging.conf')
+        sys.stdout.write('OK\n')
 
     # ## if -T run doctests (no cron)
     if hasattr(options, 'test') and options.test:

web2py.py

@@ -3,6 +3,8 @@
 
 import os
 import sys
+from multiprocessing import freeze_support
+# import gluon.import_all ##### This should be uncommented for py2exe.py
 
 if hasattr(sys, 'frozen'):
     path = os.path.dirname(os.path.abspath(sys.executable))  # for py2exe
@@ -14,17 +16,14 @@ os.chdir(path)
 
 sys.path = [path] + [p for p in sys.path if not p == path]
 
-# import gluon.import_all ##### This should be uncommented for py2exe.py
+# important that this import is after the os.chdir
+
 import gluon.widget
 
 # Start Web2py and Web2py cron service!
 if __name__ == '__main__':
-    try:
-        from multiprocessing import freeze_support
-        freeze_support()
-    except:
-        sys.stderr.write('Sorry, -K only supported for python 2.6-2.7\n')
-    if os.environ.has_key("COVERAGE_PROCESS_START"):
+    freeze_support()
+    if 'COVERAGE_PROCESS_START' in os.environ:
         try:
             import coverage
             coverage.process_startup()